# gaitsetpy.preprocessing.pipeline

This file contains functions for preprocessing the data.

Clipping, Noise Removal etc.

We'll have the following functions: clip_sliding_windows, remove_noise, remove_outliers, remove_baseline, remove_drift, remove_artifacts, remove_trend, remove_dc_offset, remove_high_frequency_noise, remove_low_frequency_noise

View Source

     1'''
     2This file contains functions for preprocessing the data.
     3
     4Clipping, Noise Removal etc.
     5
     6We'll have the following functions:
     7clip_sliding_windows, remove_noise, remove_outliers, remove_baseline, remove_drift, remove_artifacts, remove_trend, remove_dc_offset, remove_high_frequency_noise, remove_low_frequency_noise
     8'''
     9
    10import numpy as np
    11import pandas as pd
    12from scipy.signal import butter, filtfilt
    13
    14def clip_sliding_windows(data, min_val=-1, max_val=1):
    15    """
    16    Clip values in the sliding windows to be within a specified range.
    17    """
    18    return np.clip(data, min_val, max_val)
    19
    20def remove_noise(data, window_size=5):
    21    """
    22    Apply a moving average filter to reduce noise.
    23    """
    24    return data.rolling(window=window_size, center=True).mean().fillna(method="bfill").fillna(method="ffill")
    25
    26def remove_outliers(data, threshold=3):
    27    """
    28    Remove outliers beyond a given threshold using the Z-score method.
    29    """
    30    mean, std = data.mean(), data.std()
    31    return data[(data - mean).abs() <= threshold * std]
    32
    33def remove_baseline(data):
    34    """
    35    Remove baseline by subtracting the mean.
    36    """
    37    return data - data.mean()
    38
    39def remove_drift(data, cutoff=0.01, fs=100):
    40    """
    41    Remove low-frequency drift using a high-pass filter.
    42    """
    43    b, a = butter(1, cutoff / (fs / 2), btype='highpass')
    44    return filtfilt(b, a, data)
    45
    46def remove_artifacts(data, method="interpolate"):
    47    """
    48    Remove artifacts by interpolating missing values.
    49    """
    50    return data.interpolate(method="linear").fillna(method="bfill").fillna(method="ffill")
    51
    52def remove_trend(data, order=2):
    53    """
    54    Remove trends using polynomial fitting.
    55    """
    56    x = np.arange(len(data))
    57    poly_coeffs = np.polyfit(x, data, order)
    58    trend = np.polyval(poly_coeffs, x)
    59    return data - trend
    60
    61def remove_dc_offset(data):
    62    """
    63    Remove DC offset by subtracting the mean.
    64    """
    65    return data - data.mean()
    66
    67def remove_high_frequency_noise(data, cutoff=10, fs=100):
    68    """
    69    Apply a low-pass filter to remove high-frequency noise.
    70    """
    71    b, a = butter(1, cutoff / (fs / 2), btype='lowpass')
    72    return filtfilt(b, a, data)
    73
    74def remove_low_frequency_noise(data, cutoff=0.5, fs=100):
    75    """
    76    Apply a high-pass filter to remove low-frequency noise.
    77    """
    78    b, a = butter(1, cutoff / (fs / 2), btype='highpass')
    79    return filtfilt(b, a, data)

def clip_sliding_windows(data, min_val=-1, max_val=1): View Source

    15def clip_sliding_windows(data, min_val=-1, max_val=1):
    16    """
    17    Clip values in the sliding windows to be within a specified range.
    18    """
    19    return np.clip(data, min_val, max_val)

Clip values in the sliding windows to be within a specified range.

def remove_noise(data, window_size=5): View Source

    21def remove_noise(data, window_size=5):
    22    """
    23    Apply a moving average filter to reduce noise.
    24    """
    25    return data.rolling(window=window_size, center=True).mean().fillna(method="bfill").fillna(method="ffill")

Apply a moving average filter to reduce noise.

def remove_outliers(data, threshold=3): View Source

    27def remove_outliers(data, threshold=3):
    28    """
    29    Remove outliers beyond a given threshold using the Z-score method.
    30    """
    31    mean, std = data.mean(), data.std()
    32    return data[(data - mean).abs() <= threshold * std]

Remove outliers beyond a given threshold using the Z-score method.

def remove_baseline(data): View Source

    34def remove_baseline(data):
    35    """
    36    Remove baseline by subtracting the mean.
    37    """
    38    return data - data.mean()

Remove baseline by subtracting the mean.

def remove_drift(data, cutoff=0.01, fs=100): View Source

    40def remove_drift(data, cutoff=0.01, fs=100):
    41    """
    42    Remove low-frequency drift using a high-pass filter.
    43    """
    44    b, a = butter(1, cutoff / (fs / 2), btype='highpass')
    45    return filtfilt(b, a, data)

Remove low-frequency drift using a high-pass filter.

def remove_artifacts(data, method='interpolate'): View Source

    47def remove_artifacts(data, method="interpolate"):
    48    """
    49    Remove artifacts by interpolating missing values.
    50    """
    51    return data.interpolate(method="linear").fillna(method="bfill").fillna(method="ffill")

Remove artifacts by interpolating missing values.

def remove_trend(data, order=2): View Source

    53def remove_trend(data, order=2):
    54    """
    55    Remove trends using polynomial fitting.
    56    """
    57    x = np.arange(len(data))
    58    poly_coeffs = np.polyfit(x, data, order)
    59    trend = np.polyval(poly_coeffs, x)
    60    return data - trend

Remove trends using polynomial fitting.

def remove_dc_offset(data): View Source

    62def remove_dc_offset(data):
    63    """
    64    Remove DC offset by subtracting the mean.
    65    """
    66    return data - data.mean()

Remove DC offset by subtracting the mean.

def remove_high_frequency_noise(data, cutoff=10, fs=100): View Source

    68def remove_high_frequency_noise(data, cutoff=10, fs=100):
    69    """
    70    Apply a low-pass filter to remove high-frequency noise.
    71    """
    72    b, a = butter(1, cutoff / (fs / 2), btype='lowpass')
    73    return filtfilt(b, a, data)

Apply a low-pass filter to remove high-frequency noise.

def remove_low_frequency_noise(data, cutoff=0.5, fs=100): View Source

    75def remove_low_frequency_noise(data, cutoff=0.5, fs=100):
    76    """
    77    Apply a high-pass filter to remove low-frequency noise.
    78    """
    79    b, a = butter(1, cutoff / (fs / 2), btype='highpass')
    80    return filtfilt(b, a, data)

Apply a high-pass filter to remove low-frequency noise.