im: add project files

.gitignore

+__pycache__
+.idea
+/assets/
+build
+dist
+*.iec
+*.txt
+*.h5
+*.pkl
+/venv/
+/iec/
+/results/

blink_detection_classifier.py

+import numpy as np
+import pickle
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
+from sklearn.svm import SVC
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.preprocessing import StandardScaler
+from lazypredict.Supervised import LazyClassifier
+from scipy import signal, stats
+import seaborn as sns
+import matplotlib.pyplot as plt
+
+from server.sl8 import FileBase
+from extra import design_filters
+
+
+def feature_extraction(data_):
+    # data_ = data_ + abs(np.min(data_))
+    # ave = np.mean(data_)
+    var = np.var(data_)
+    std = np.sqrt(var)
+    rms = np.sqrt(np.mean(data_ ** 2))
+    skewness = stats.skew(data_)
+    kurtosis = stats.kurtosis(data_, fisher=False)
+    ptop = np.max(data_) - np.min(data_)
+    auc = int(np.trapz(data_, dx=1))
+    zero_crossings = np.where(np.diff(np.sign(data_)))[0]
+
+    # f, pxx = signal.welch(data_, fs=sr,
+    #                       window=signal.windows.tukey(len(data_), sym=False, alpha=.17),
+    #                       nperseg=len(data_), noverlap=int(len(data_) * 0.75),
+    #                       nfft=2 * sr, return_onesided=True,
+    #                       scaling='spectrum', axis=-1, average='mean')
+    # psd = np.sum(np.log10(pxx[2:6]))
+
+    # print(ave, var, std, rms, skewness, kurtosis, ptop, auc, len(zero_crossings), psd)
+    return [var, std, rms, skewness, kurtosis, ptop, auc, len(zero_crossings)]
+    # return [var, std, rms, skewness, kurtosis, ptop, psd]
+
+
+# filename = './iec_files/1615632656_EyeOpen_1_Other.iec'
+# filename = './iec_files/1618321042_EyeOpen_6_sina_izani.iec'
+# filename = './iec_files/1618747412_EyeOpen_7_melika_hoseinzadeh.iec'
+# filename = './iec_files/1619683416_EyeOpen_9_khosravi.iec'
+# filename = './iec_files/1620309313_EyeOpen_12_mr_mohsen_shahbaz_beigi.iec'
+# filename = './iec_files/1620576311_EyeOpen_14_mahan_endallah.iec'
+# filename = './iec_files/1621949668_EyeOpen_16_diyana_ebrahimi_0.iec'
+# filename = './iec_files/1623149547_EyeOpen_1_Other_3.iec'
+# filename = './iec_files/1623154705_EyeOpen_1_Other_3.iec'
+# filename = './iec_files/1623597049_EyeOpen_18_anita_mirzayi_0.iec'
+# filename = './iec_files/1624281526_EyeOpen_1_Other_0.iec'
+# filename = './iec_files/1624281967_EyeOpen_1_Other_0.iec'
+# file_list = [filename]
+
+file_list = ['./iec_files/1615632656_EyeOpen_1_Other.iec',
+             './iec_files/1618321042_EyeOpen_6_sina_izani.iec',
+             './iec_files/1618747412_EyeOpen_7_melika_hoseinzadeh.iec',
+             './iec_files/1619683416_EyeOpen_9_khosravi.iec',
+             './iec_files/1620309313_EyeOpen_12_mr_mohsen_shahbaz_beigi.iec',
+             './iec_files/1620576311_EyeOpen_14_mahan_endallah.iec',
+             './iec_files/1621949668_EyeOpen_16_diyana_ebrahimi_0.iec',
+             './iec_files/1623149547_EyeOpen_1_Other_3.iec',
+             './iec_files/1623154705_EyeOpen_1_Other_3.iec',
+             './iec_files/1623597049_EyeOpen_18_anita_mirzayi_0.iec',
+             './iec_files/1624281526_EyeOpen_1_Other_0.iec',
+             './iec_files/1624281967_EyeOpen_1_Other_0.iec'
+             ]
+
+feature_list = []
+for filename in file_list:
+    labeled_filename = filename.replace('iec_files', 'label_results').replace('iec', 'txt')
+
+    iec = FileBase()
+    iec.load(filename)
+    data = iec.get_brain_raw_data()
+    sr = iec.get_brain_sample_rate()
+    gain = iec.get_gain()
+
+    data = data / gain
+    data = signal.detrend(data, axis=1)
+    filters = design_filters(sr)
+    data = signal.filtfilt(filters['notch1'][0], filters['notch1'][1], data)
+    data = signal.filtfilt(filters['notch2'][0], filters['notch2'][1], data)
+    data = signal.filtfilt(filters['bandpass'][0], filters['bandpass'][1], data)
+
+    # with open(labeled_filename) as f:
+    #     content = f.readlines()
+    # # you may also want to remove whitespace characters like `\n` at the end of each line
+    # content = [x.strip()[1:-1] for x in content]
+    #
+    # arr = np.zeros((len(content), 4), dtype=int)
+    # for i in range(len(content)):
+    #     temp = content[i].split(',')
+    #     arr[i, 0] = int(temp[0])
+    #     arr[i, 1] = int(temp[1])
+    #     arr[i, 2] = int(temp[2])
+    #     arr[i, 3] = int(temp[3])
+    # del content
+    arr = np.loadtxt(labeled_filename, dtype=int)
+
+    features = []
+    for i in range(arr.shape[0]):
+        # print(arr[i, 0], arr[i, 1], arr[i, 2])
+        my_data = data[arr[i, 0], arr[i, 1]: arr[i, 2]]
+        # my_data = my_data / gain
+        # my_data -= np.mean(my_data)
+        # my_data = signal.detrend(my_data)
+
+        # my_data = signal.filtfilt(filters['notch1'][0], filters['notch1'][1], my_data)
+        # my_data = signal.filtfilt(filters['notch2'][0], filters['notch2'][1], my_data)
+        # my_data = signal.filtfilt(filters['bandpass'][0], filters['bandpass'][1], my_data)
+
+        # my_data = (my_data - np.min(my_data)) / (np.max(my_data) - np.min(my_data))
+        # my_data = my_data / np.max(my_data)
+        # my_data = (my_data - np.min(my_data)) / np.ptp(my_data)
+
+        feats = feature_extraction(my_data)
+        feats.append(arr[i, 3])
+        features.append(feats)
+
+    features = np.array(features)
+
+    features = features[features[:, -1].argsort()]
+    a = features[features[:, -1] == 0]
+    b = features[features[:, -1] == 1]
+    c = a[np.random.randint(a.shape[0], size=b.shape[0]), :]
+    features = np.vstack((c, b))
+    feature_list.append(features)
+
+X_all = feature_list[0]
+for i in range(0, len(feature_list) - 1):
+    X_all = np.concatenate((X_all, feature_list[i + 1]), axis=0)
+
+X = X_all[:, :-1]
+y = X_all[:, -1]
+
+a = y[y == 0]
+b = y[y == 1]
+print(len(a), len(b))
+
+scaler = StandardScaler()
+scaled_data = scaler.fit_transform(X)
+
+X_train, X_test, y_train, y_test = train_test_split(scaled_data, y, test_size=0.30)
+
+svclassifier = SVC(kernel='rbf', class_weight={1: len(a) / len(b)}, probability=True)
+svclassifier.fit(X_train, y_train)
+
+y_pred = svclassifier.predict(X_test)
+
+cm = confusion_matrix(y_test, y_pred)
+acc = accuracy_score(y_test, y_pred)
+print('Accuracy:', acc)
+print(cm)
+print(classification_report(y_test, y_pred))
+plt.figure()
+ax = plt.subplot()
+plt.suptitle('Accuracy: ' + str(acc))
+sns.heatmap(cm, annot=True, ax=ax, fmt='g', cmap='Greens')
+plt.show()
+
+# # save the classifier
+# model = {'model': svclassifier, 'scaler': scaler}
+# with open('svclassifier_new3.pkl', 'wb') as fid:
+#     pickle.dump(model, fid)
+
+# rfc = RandomForestClassifier(max_depth=10, random_state=0)
+# rfc.fit(X_train, y_train)
+#
+# y_pred = rfc.predict(X_test)
+#
+# cm = confusion_matrix(y_test, y_pred)
+# acc = accuracy_score(y_test, y_pred)
+# print('Accuracy:', acc)
+# print(cm)
+# print(classification_report(y_test, y_pred))
+# plt.figure()
+# ax = plt.subplot()
+# plt.suptitle('Accuracy: ' + str(acc))
+# sns.heatmap(cm, annot=True, ax=ax, fmt='g', cmap='Greens')
+# plt.show()
+
+# clf = LazyClassifier(verbose=0, ignore_warnings=True, custom_metric=None)
+# models, predictions = clf.fit(X_train, X_test, y_train, y_test)
+# print(models)

blink_remove.py

+import numpy as np
+from scipy import signal, stats
+from sklearn.decomposition import FastICA
+from pywt import wavedec
+from pywt import waverec
+import matplotlib.pyplot as plt
+from matplotlib import colors as mcolors
+from matplotlib.collections import LineCollection
+
+
+def blinkremove(eeg_signal, srate, chnum, debug_mode=False):
+    """
+    removes blink artifact from signal
+    :param eeg_signal: input eeg signal
+    :param srate: sampling rate
+    :return: blink removed signal
+    """
+
+    ############### ICA ######################
+    # the length of samples must be even
+    del_flag = 0
+    if (np.shape(eeg_signal)[1]) % 2 != 0:
+        eeg_signal = np.delete(eeg_signal, 0, 1)
+        del_flag = 1
+
+    # chnum = eeg_signal.shape[0]
+    eeg_signal = eeg_signal.transpose()
+    ica = FastICA(n_components=chnum, whiten=True)
+
+    eeg_unmixed = ica.fit_transform(eeg_signal)  # Reconstruct signals
+    iteres = ica.n_iter_
+    print("iteration needed:", iteres)
+    if iteres > 199:
+        return 'converge error'
+    myvar = np.sqrt(np.var(eeg_unmixed))
+    eeg_unmixed = eeg_unmixed / myvar  # this is to handle the scikit learn bug!
+    A_ = ica.mixing_  # Get estimated mixing matrix
+    eeg_unmixed = eeg_unmixed.transpose()
+    if debug_mode:
+        eegplot(eeg_unmixed, srate, 'ica unmixed signal')
+
+    ############### Kurtosis ######################
+    Kurtosis = stats.kurtosis(eeg_unmixed, axis=-1)
+
+    # thresholding
+    lessKurt = Kurtosis.copy()
+    # # throw maximum away and measure std with others
+    # maxIndex = np.where(lessKurt == max(lessKurt))
+    # maxIndex = maxIndex[0][0]
+    # lessKurt = np.array(lessKurt, maxIndex)
+    mKurt = np.mean(lessKurt)
+    sdKurt = np.std(lessKurt)
+    CI = 0.90  # two tail 0.9
+    tN_1 = stats.t.ppf(1 - (1 - CI) / 2, chnum - 1)
+    kuUpperLim = mKurt + sdKurt / np.sqrt(chnum) * tN_1
+
+    if debug_mode:
+        plt.figure('Kurtosis of ICs')
+        plt.bar(list(range(1, chnum + 1)), Kurtosis)
+        plt.title('Kurtosis of ICs')
+        plt.plot(list(range(0, chnum + 2)), kuUpperLim * np.ones(chnum + 2), 'r', 'LineWidth', 1)
+
+    ############### Skewness ######################
+    skew = stats.skew(eeg_unmixed, axis=-1)
+    # thresholding
+    lessskew = skew.copy()
+    mskew = np.mean(lessskew)
+    sdskew = np.std(lessskew)
+    CI = 0.95  # two tail 0.95
+    tN_1 = stats.t.ppf(1 - (1 - CI) / 2, chnum - 1)
+    skUpperLim = mskew + sdskew / np.sqrt(chnum) * tN_1
+    skLowerLim = mskew - sdskew / np.sqrt(chnum) * tN_1
+
+    if debug_mode:
+        plt.figure('Skweness of ICs')
+        plt.bar(list(range(1, chnum + 1)), skew)
+        plt.title('Skweness of ICs')
+
+        plt.plot(list(range(0, chnum + 2)), skUpperLim * np.ones(chnum + 2), 'r', 'LineWidth', 1)
+        plt.plot(list(range(0, chnum + 2)), skLowerLim * np.ones(chnum + 2), 'r', 'LineWidth', 1)
+
+    ############### Perform Wavelet on Blink Components ######################
+    isblink = (skew > skUpperLim) | (skew < skLowerLim) | (Kurtosis > kuUpperLim)
+
+    eeg_ica_noblink = eeg_unmixed.copy()
+
+    # for each blink detected IC
+    for xcomp in range(chnum):
+        if isblink[xcomp]:
+            blink = eeg_unmixed[xcomp, :]
+            coeffs = wavedec(data=blink, level=8, wavelet='bior4.4')
+
+            # null the blink wavelet coefficients
+            newcoeffs = coeffs.copy()
+            K = np.zeros(9)
+            for ix in range(9):
+                temp = coeffs[ix]
+                sigma2 = np.median(abs(temp) / 0.6745)
+                K[ix] = np.sqrt(2 * np.log10(len(temp)) * sigma2)
+                # type I
+                Indx = list(np.where(np.abs(temp) > K[ix])[0])
+                temp[Indx] = 0
+                newcoeffs[ix] = temp
+
+            eeg_ica_noblink[xcomp] = waverec(newcoeffs, 'bior4.4')
+
+    eeg_ica_noblink = np.array(eeg_ica_noblink)
+    if debug_mode:
+        eegplot(eeg_ica_noblink, srate, 'ICA Blink Removed')
+
+    ############### Reconstruct EEG signals #######################
+
+    # inverse of ICA
+    eeg_ica_noblink = eeg_ica_noblink * myvar  # this is to handle the scikit learn bug!
+    eeg_ica_noblink = eeg_ica_noblink.transpose()
+    eeg_blink_removed = ica.inverse_transform(eeg_ica_noblink)
+    eeg_blink_removed = eeg_blink_removed.transpose()
+    eeg_signal = eeg_signal.transpose()
+    # Spectral Coherence Measure
+    if debug_mode:
+        cxy = []
+        for xch in range(chnum):
+            # [cxy(xChannel,:), fc] = mscohere(data(xChannel,:), FinalEEG(xChannel,:), hamming(512), 256, 256, 2000)
+            f, Cxy = signal.coherence(eeg_signal[xch], eeg_blink_removed[xch], srate, nperseg=1024)
+            cxy.append(Cxy)
+
+        mcxy = np.mean(np.array(cxy), axis=0)
+        plt.figure("Spectral Coherence Measurement")
+        plt.stem(f, mcxy)
+
+    if del_flag == 1:
+        eeg_blink_removed = np.hstack((eeg_blink_removed, np.tile(eeg_blink_removed[:, [-1]], 1)))
+    return eeg_blink_removed
+
+
+def eegplot(eeg_data, srate, title, fig=None, block=False):
+    """
+    the function to plot eeg data
+    :param eeg_data: get eeg data
+    :param srate: sampling rate of eeg signal
+    :param title: string of plot title
+    :return: plot the data
+    """
+    eeg_data = np.array(eeg_data)
+    eeg_data[:-1, :] = signal.detrend(eeg_data[:-1, :], axis=-1, type='linear')
+    n_samples, n_rows = np.shape(eeg_data)[1], np.shape(eeg_data)[0]
+
+    # normalize each column to be able to show
+    for i in range(n_rows):
+        eeg_data[i, :] = eeg_data[i, :] - np.mean(eeg_data[i, :])
+        eeg_data[i, :] = eeg_data[i, :] / (np.std(eeg_data[i, :]) + 1)
+
+    if fig is None:
+        fig = plt.figure("EEG plot of {}".format(title))
+
+    axes = plt.axes()
+    # Load the EEG data
+    data = np.transpose(eeg_data)
+    t = np.arange(n_samples) / srate
+
+    # Plot the EEG
+    ticklocs = []
+    axes.set_xlim(0, t.max())
+    # ax2.set_xticks(np.arange(10))
+
+    segs = []
+    y1 = 0
+    for i in range(n_rows):
+        dmin = data[:, i].min()
+        dmax = data[:, i].max()
+        dr = (dmax - dmin)
+        segs.append(np.column_stack((t, data[:, i])))
+        ticklocs.append(y1)
+        y1 = y1 + dr
+
+    y0 = data[:, 0].min()
+    axes.set_ylim(y0, y1)
+
+    offsets = np.zeros((n_rows, 2), dtype=float)
+    offsets[:, 1] = ticklocs
+
+    colors = [mcolors.to_rgba(c)
+              for c in plt.rcParams['axes.prop_cycle'].by_key()['color']]
+    lines = LineCollection(segs, offsets=offsets, transOffset=None, linewidths=0.5,
+                           colors=colors, linestyle='solid')
+    axes.add_collection(lines)
+
+    # Set the yticks to use axes coordinates on the y axis
+    axes.set_yticks(ticklocs)
+
+    ch_name_list = []
+    for i in range(n_rows):
+        ch_name_list.append('CH '+str(i))
+    axes.set_yticklabels(ch_name_list)
+
+    axes.set_xlabel('Time (s)')
+    plt.suptitle(title)
+
+    plt.tight_layout()
+    plt.show()

blink_remove_run.py

+import numpy as np
+
+from blink_remove import blinkremove, eegplot
+from scipy import signal
+
+from server.sl8 import FileBase
+from extra import design_filters, iec_maker
+
+file_list = []
+
+for filename in file_list:
+    try:
+        iec = FileBase()
+        iec.load(filename)
+        data = iec.get_brain_raw_data()
+        sr = iec.get_brain_sample_rate()
+        chnum = iec.get_brain_channel_count()
+        gain = iec.get_gain()
+        lead_off = iec.get_leadoff()
+
+        filters = design_filters(sr)
+        data = signal.filtfilt(filters['notch1'][0], filters['notch1'][1], data)
+        data = signal.filtfilt(filters['notch2'][0], filters['notch2'][1], data)
+        data = signal.filtfilt(filters['bandpass'][0], filters['bandpass'][1], data)
+
+        data = data[:, 1024:-1024]
+
+        eegplot(data, sr, 'EEG before ICA')
+        clean_data = blinkremove(data, sr, chnum, debug_mode=False)
+        eegplot(clean_data, sr, 'EEG after ICA')
+
+        iec_maker(clean_data, filename.split('/')[-1].split('.')[0], sr)
+    except Exception as e:
+        print(e)

denoise_conv_2d.py

+import numpy as np
+import matplotlib.pyplot as plt
+import tensorflow as tf
+from keras.datasets.fashion_mnist import load_data
+from keras.models import Sequential
+from keras.layers import Conv2D, Conv2DTranspose, MaxPool2D, GaussianNoise, Reshape
+from keras.optimizers import SGD, RMSprop, Adam, Adadelta, Adamax, Adagrad
+
+(X_train_full, y_train_full), (X_test, y_test) = load_data()
+X_train_full = X_train_full.astype(np.float32) / 255
+X_test = X_test.astype(np.float32) / 255
+X_train, X_valid = X_train_full[:55000], X_train_full[55000:]
+# y_train, y_valid = y_train_full[:55000], y_train_full[55000:]
+
+# targets = ["T-shirt/top", "Trouser", "Pullover", "Dress", "Coat", "Sandal",
+#            "Shirt", "Sneaker", "Bag", "Ankle boot"]
+# n_rows = 3
+# n_cols = 8
+# plt.figure(figsize=(n_cols * 2, n_rows * 2))
+# for row in range(n_rows):
+#     for col in range(n_cols):
+#         index = n_cols * row + col
+#         plt.subplot(n_rows, n_cols, index + 1)
+#         plt.imshow(X_train[index], cmap="binary")
+#         plt.axis('off')
+#         plt.title(targets[y_train[index]], fontsize=12)
+# plt.subplots_adjust(wspace=0.2, hspace=0.5)
+# plt.show()
+
+denoising_encoder = Sequential([
+    Reshape([28, 28, 1], input_shape=[28, 28]),
+    GaussianNoise(0.2),
+    Conv2D(16, kernel_size=3, padding="SAME", activation="selu"),
+    MaxPool2D(pool_size=2),
+    Conv2D(32, kernel_size=3, padding="SAME", activation="selu"),
+    MaxPool2D(pool_size=2),
+    Conv2D(64, kernel_size=3, padding="SAME", activation="selu"),
+    MaxPool2D(pool_size=2)
+])
+
+# tf.keras.utils.plot_model(denoising_encoder, to_file='denoising_encoder.png', show_shapes=True)
+
+denoising_decoder = Sequential([
+    Conv2DTranspose(32, kernel_size=3, strides=2, padding="VALID", activation="selu",
+                    input_shape=[3, 3, 64]),
+    Conv2DTranspose(16, kernel_size=3, strides=2, padding="SAME", activation="selu"),
+    Conv2DTranspose(1, kernel_size=3, strides=2, padding="SAME", activation="sigmoid"),
+    Reshape([28, 28])
+])
+
+# tf.keras.utils.plot_model(denoising_decoder, to_file='denoising_decoder.png', show_shapes=True)
+
+denoising_ae = Sequential([denoising_encoder, denoising_decoder])
+denoising_ae.compile(loss="binary_crossentropy", optimizer=SGD(lr=0.5))
+history = denoising_ae.fit(X_train, X_train, epochs=5,
+                           validation_data=(X_valid, X_valid))
+
+corrupted_inputs = GaussianNoise(0.2)(X_valid[8:13], training=True)
+reconstructs = denoising_ae.predict(corrupted_inputs)
+plt.figure(figsize=(15, 12))
+for i in range(5):
+    plt.subplot(2, 5, 1 + i)
+    plt.imshow(corrupted_inputs[i], cmap='binary')
+    plt.axis('off')
+    plt.subplot(2, 5, 6 + i)
+    plt.imshow(reconstructs[i], cmap='binary')
+    plt.axis('off')
+plt.show()

denoise_with_dense.py

+from keras.datasets import mnist
+from keras.layers import Input, Dense
+from keras.models import Model
+import numpy as np
+import matplotlib.pyplot as plt
+
+# loading only images and not their labels
+(X_train, _), (X_test, _) = mnist.load_data()
+
+X_train = X_train.astype('float32') / 255
+X_test = X_test.astype('float32') / 255
+
+X_train = X_train.reshape(len(X_train), np.prod(X_train.shape[1:]))
+X_test = X_test.reshape(len(X_test), np.prod(X_test.shape[1:]))
+X_train_noisy = X_train + np.random.normal(loc=0.0, scale=0.5, size=X_train.shape)
+X_train_noisy = np.clip(X_train_noisy, 0., 1.)
+X_test_noisy = X_test + np.random.normal(loc=0.0, scale=0.5, size=X_test.shape)
+X_test_noisy = np.clip(X_test_noisy, 0., 1.)
+print(X_train_noisy.shape)
+print(X_test_noisy.shape)
+
+# Input image
+input_img = Input(shape=(784,))
+# encoded and decoded layer for the autoencoder
+encoded = Dense(units=128, activation='relu')(input_img)
+encoded = Dense(units=64, activation='relu')(encoded)
+encoded = Dense(units=32, activation='relu')(encoded)
+decoded = Dense(units=64, activation='relu')(encoded)
+decoded = Dense(units=128, activation='relu')(decoded)
+decoded = Dense(units=784, activation='sigmoid')(decoded)
+# Building autoencoder
+autoencoder = Model(input_img, decoded)
+# extracting encoder
+encoder = Model(input_img, encoded)
+# compiling the autoencoder
+autoencoder.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
+
+# Fitting the noise trained data to the autoencoder
+# autoencoder.fit(X_train_noisy, X_train_noisy,
+#                 epochs=100,
+#                 batch_size=128,
+#                 shuffle=True,
+#                 validation_data=(X_test_noisy, X_test_noisy))
+
+autoencoder.fit(X_train_noisy, X_train,
+                epochs=100,
+                batch_size=128,
+                validation_split=0.2,
+                verbose=2)
+
+# reconstructing the image from autoencoder and encoder
+encoded_imgs = encoder.predict(X_test_noisy)
+predicted = autoencoder.predict(X_test_noisy)
+# plotting the noised image, encoded image and the reconstructed image
+plt.figure(figsize=(20, 2))
+for i in range(10):
+    # display original images
+
+    ax = plt.subplot(4, 20, i + 1)
+    plt.imshow(X_test[i].reshape(28, 28))
+    plt.gray()
+    ax.get_xaxis().set_visible(False)
+    ax.get_yaxis().set_visible(False)
+
+    # display noised images
+    ax = plt.subplot(4, 20, i + 1 + 20)
+    plt.imshow(X_test_noisy[i].reshape(28, 28))
+    plt.gray()
+    ax.get_xaxis().set_visible(False)
+    ax.get_yaxis().set_visible(False)
+    # display encoded images
+    ax = plt.subplot(4, 20, 2 * 20 + i + 1)
+    plt.imshow(encoded_imgs[i].reshape(8, 4))
+    plt.gray()
+    ax.get_xaxis().set_visible(False)
+    ax.get_yaxis().set_visible(False)
+    # display reconstruction images
+    ax = plt.subplot(4, 20, 3 * 20 + i + 1)
+    plt.imshow(predicted[i].reshape(28, 28))
+    plt.gray()
+    ax.get_xaxis().set_visible(False)
+    ax.get_yaxis().set_visible(False)
+
+plt.show()

extra.py

+import numpy as np
+from scipy import signal
+
+from server.sl8 import FileBase
+
+
+def design_filters(s_rate):
+    filters = {}
+    output = signal.butter(N=5, Wn=np.array([45, 55]), btype='bandstop', analog=False, output='ba', fs=s_rate)
+    filters['notch1'] = [output[0], output[1]]  # b, a
+    output = signal.butter(N=5, Wn=np.array([95, 105]), btype='bandstop', analog=False, output='ba', fs=s_rate)
+    filters['notch2'] = [output[0], output[1]]  # b, a
+    output = signal.butter(N=5, Wn=np.array([0.53, 32]), btype='bandpass', analog=False, output='ba', fs=s_rate)
+    filters['bandpass'] = [output[0], output[1]]  # b, a
+    return filters
+
+
+# def iec_maker(data, id_, sps, gain, leadoff):
+def iec_maker(data, id_, sps):
+    identifier = str(id_)
+    sps = int(sps)
+    file_base = FileBase(subject_id=0, protocol_name='', task_name=identifier, fatigue=0,
+                         hungriness=0, sequence=[], project='1', description='')
+
+    file_base.set_brain_data(data, int(len(data)), int(len(data[0])), sps)  # All fields are numeric.
+    file_base.set_device_data('s', '12', '12', 's')  # Use standard patterns like what you can see on the server.
+    file_base.set_sensor('s')  # Use string name of sensor.
+    file_base.set_extra_signal(data, int(3), int(len(data[0])), sps)  # All fields are numeric.
+    file_base.set_output_data(data, int(len(data)), int(len(data[0])), sps)  # All fields are numeric.
+    file_base.set_reaction_data([0], [0], [0])
+    # file_base.set_gain(gain)
+    # file_base.set_leadoff(leadoff)
+    file_base.save('blink_removed_iec_cropped/', filename=id_)
+    # file_base.save('iec_files/', filename=id_)
+    return 'success'

signal_apply_noise.py

+import matplotlib.pyplot as plt
+import numpy as np
+
+# Sample configuration
+num_samples_visualize = 5
+noise_factor = 0.05
+
+# Load data
+data = np.load('./signal_waves_medium.npy')
+x_val, y_val = data[:, 0], data[:, 1]
+
+# Add noise to data
+noisy_samples = []
+for i in range(0, len(x_val)):
+    if i % 100 == 0:
+        print(i)
+    pure = np.array(y_val[i])
+    noise = np.random.normal(0, 10, pure.shape)
+    signal = pure + noise_factor * noise
+    noisy_samples.append([x_val[i], signal])
+
+# Save data to file for re-use
+np.save('./signal_waves_noisy_medium.npy', noisy_samples)
+
+# Visualize a few random samples
+for i in range(0, num_samples_visualize):
+    random_index = np.random.randint(0, len(noisy_samples) - 1)
+    x_axis, y_axis = noisy_samples[random_index]
+    plt.plot(x_axis, y_axis)
+    plt.title(f'Visualization of noisy sample {random_index} ---- y: f(x) = x^2')
+    plt.show()

signal_autoencoder.py

+from tensorflow.keras.models import Sequential, save_model
+from tensorflow.keras.layers import Conv1D, Conv1DTranspose
+from tensorflow.keras.constraints import max_norm
+import matplotlib.pyplot as plt
+import numpy as np
+import math
+
+# Model configuration
+input_shape = (150, 1)
+batch_size = 150
+no_epochs = 5
+train_test_split = 0.3
+validation_split = 0.2
+verbosity = 1
+max_norm_value = 2.0
+
+# Load data
+data_noisy = np.load('./signal_waves_noisy_medium.npy')
+x_val_noisy, y_val_noisy = data_noisy[:, 0], data_noisy[:, 1]
+data_pure = np.load('./signal_waves_medium.npy')
+x_val_pure, y_val_pure = data_pure[:, 0], data_pure[:, 1]
+
+# Reshape data
+y_val_noisy_r = []
+y_val_pure_r = []
+for i in range(0, len(y_val_noisy)):
+    noisy_sample = y_val_noisy[i]
+    pure_sample = y_val_pure[i]
+    # noisy_sample = (noisy_sample - np.min(noisy_sample)) / (np.max(noisy_sample) - np.min(noisy_sample))
+    # pure_sample = (pure_sample - np.min(pure_sample)) / (np.max(pure_sample) - np.min(pure_sample))
+    noisy_sample = noisy_sample / np.max(noisy_sample)
+    pure_sample = pure_sample / np.max(pure_sample)
+    y_val_noisy_r.append(noisy_sample)
+    y_val_pure_r.append(pure_sample)
+y_val_noisy_r = np.array(y_val_noisy_r)
+y_val_pure_r = np.array(y_val_pure_r)
+noisy_input = y_val_noisy_r.reshape((y_val_noisy_r.shape[0], y_val_noisy_r.shape[1], 1))
+pure_input = y_val_pure_r.reshape((y_val_pure_r.shape[0], y_val_pure_r.shape[1], 1))
+
+# Train/test split
+percentage_training = math.floor((1 - train_test_split) * len(noisy_input))
+noisy_input, noisy_input_test = noisy_input[:percentage_training], noisy_input[percentage_training:]
+pure_input, pure_input_test = pure_input[:percentage_training], pure_input[percentage_training:]
+
+# Create the model
+model = Sequential()
+model.add(Conv1D(150, kernel_size=3, kernel_constraint=max_norm(max_norm_value), activation='relu',
+                 kernel_initializer='he_uniform', input_shape=input_shape))
+# model.add(Conv1D(64, kernel_size=3, kernel_constraint=max_norm(max_norm_value), activation='relu',
+#                  kernel_initializer='he_uniform'))
+model.add(Conv1D(32, kernel_size=3, kernel_constraint=max_norm(max_norm_value), activation='relu',
+                 kernel_initializer='he_uniform'))
+model.add(Conv1DTranspose(32, kernel_size=3, kernel_constraint=max_norm(max_norm_value), activation='relu',
+                          kernel_initializer='he_uniform'))
+# model.add(Conv1DTranspose(64, kernel_size=3, kernel_constraint=max_norm(max_norm_value), activation='relu',
+#                           kernel_initializer='he_uniform'))
+model.add(Conv1DTranspose(150, kernel_size=3, kernel_constraint=max_norm(max_norm_value), activation='relu',
+                          kernel_initializer='he_uniform'))
+model.add(Conv1D(1, kernel_size=3, kernel_constraint=max_norm(max_norm_value), activation='sigmoid', padding='same'))
+
+model.summary()
+
+# Compile and fit data
+model.compile(optimizer='adam', loss='binary_crossentropy')
+# model.fit(noisy_input, pure_input,
+#           epochs=no_epochs,
+#           batch_size=batch_size,
+#           validation_split=validation_split,
+#           verbose=verbosity)
+model.fit(x=noisy_input, y=pure_input,
+          epochs=no_epochs,
+          batch_size=batch_size,
+          verbose=verbosity,
+          shuffle=True,
+          validation_data=(noisy_input_test, pure_input_test))
+
+# Generate reconstructions
+num_reconstructions = 4
+samples = noisy_input_test[:num_reconstructions]
+reconstructions = model.predict(samples)
+
+# Plot reconstructions
+for i in np.arange(0, num_reconstructions):
+    # Prediction index
+    prediction_index = i + percentage_training
+    # Get the sample and the reconstruction
+    original = y_val_noisy[prediction_index]
+    pure = y_val_pure[prediction_index]
+    reconstruction = np.array(reconstructions[i])
+    # Matplotlib preparations
+    fig, axes = plt.subplots(1, 3)
+    # Plot sample and reconstruciton
+    axes[0].plot(original)
+    axes[0].set_title('Noisy waveform')
+    axes[1].plot(pure)
+    axes[1].set_title('Pure waveform')
+    axes[2].plot(reconstruction)
+    axes[2].set_title('Conv Autoencoder Denoised waveform')
+    plt.show()

signal_generator.py

+import matplotlib.pyplot as plt
+import numpy as np
+
+# Sample configuration
+num_samples = 100000
+
+# Intrasample configuration
+num_elements = 1
+interval_per_element = 0.01
+total_num_elements = int(num_elements / interval_per_element)
+starting_point = int(0 - 1 * total_num_elements)
+
+# Other configuration
+num_samples_visualize = 1
+
+# Containers for samples and subsamples
+samples = []
+xs = []
+ys = []
+
+# Generate samples
+for j in range(0, num_samples):
+    # Report progress
+    if j % 100 == 0:
+        print(j)
+    # Generate wave
+    for i in range(starting_point, total_num_elements):
+        x_val = i * interval_per_element
+        y_val = x_val * x_val
+        xs.append(x_val)
+        ys.append(y_val)
+    # Append wave to samples
+    samples.append((xs, ys))
+    # Clear subsample containers for next sample
+    xs = []
+    ys = []
+
+# Input shape
+print(np.shape(np.array(samples[0][0])))
+
+# Save data to file for re-use
+np.save('./signal_waves_medium.npy', samples)
+
+# Visualize a few random samples
+for i in range(0, num_samples_visualize):
+    random_index = np.random.randint(0, len(samples) - 1)
+    x_axis, y_axis = samples[random_index]
+    plt.plot(x_axis, y_axis)
+    plt.title(f'Visualization of sample {random_index} ---- y: f(x) = x^2')
+    plt.show()