1import pandas as pd
 2import numpy as np
 3from datetime import datetime
 4import matplotlib.pyplot as plt
 5
 6# Simulate mock energy production dataset
 7def simulate_energy_data(start_date, end_date, freq='10min'):
 8    # Create a datetime index with 10-minute intervals
 9    datetime_index = pd.date_range(start=start_date, end=end_date, freq=freq)
10
11    # Simulate energy production with day-night cycles
12    np.random.seed(42)  # For reproducibility
13    hours = datetime_index.hour
14    day_energy = np.random.normal(loc=300, scale=30, size=len(hours))
15    night_energy = np.random.normal(loc=50, scale=15, size=len(hours))
16    energy_values = np.where((hours >= 6) & (hours <= 18), day_energy, night_energy)
17
18    # Introduce missing values (10% of the dataset)
19    num_missing = int(0.1 * len(energy_values))  # 10% of data will be missing
20    missing_indices = np.random.choice(len(energy_values), num_missing, replace=False)
21    energy_values[missing_indices] = np.nan  # Set randomly selected indices to NaN
22
23    # Create DataFrame with the simulated energy data
24    energy_data = pd.DataFrame({
25        'Datetime': datetime_index,
26        'Energy_Production': energy_values
27    })
28
29    return energy_data
30
31
32def plot_origin_data(energy_data_with_missing):
33    plt.figure(figsize=(24, 6))
34    plt.plot(energy_data_with_missing['Datetime'], energy_data_with_missing['Energy_Production'], label='Energy Production')
35    plt.xlabel('Datetime')
36    plt.ylabel('Energy Production')
37    plt.title('Simulated Energy Production Over Time')
38    plt.legend()
39
40    plt.grid(True)
41    plt.show()
42# Main script
43if __name__ == "__main__":
44    start_date = datetime(2025, 1, 1)
45    end_date = datetime(2025, 4, 40)
46
47    # Generate the simulated dataset
48    energy_data_with_missing = simulate_energy_data(start_date, end_date)
49
50    # Display the first few rows of the dataset
51    print(energy_data_with_missing.head())
52
53
54    plot_origin_data(energy_data_with_missing)

 1
 2from sklearn.tree import DecisionTreeRegressor
 3
 4# Impute missing values using linear regression
 5def impute_missing_values(data):
 6    # Extract the non-missing data
 7    non_missing_data = data.dropna()
 8
 9    # Prepare the features (X) and target (y)
10    X = non_missing_data.index.values.reshape(-1, 1)
11    y = non_missing_data['Energy_Production'].values
12
13    # Fit the linear regression model
14    model =   DecisionTreeRegressor(max_depth=5, random_state=42)
15    model.fit(X, y)
16
17    # Predict the missing values
18    missing_data = data[data['Energy_Production'].isna()]
19    X_missing = missing_data.index.values.reshape(-1, 1)
20    predicted_values = model.predict(X_missing)
21
22    # Create a DataFrame for the imputed data
23    imputed_data = data.copy()
24    imputed_data.loc[data['Energy_Production'].isna(), 'Energy_Production'] = predicted_values
25
26    return imputed_data
27
28
29def plot_full_data(energy_data_imputed,energy_data_with_missing):
30    plt.figure(figsize=(24, 6))
31
32    plt.plot(energy_data_imputed['Datetime'], energy_data_imputed['Energy_Production'], label='Imputed Data using Decision regression', color='blue' )
33    plt.plot(energy_data_with_missing['Datetime'], energy_data_with_missing['Energy_Production'], label='Original Data', color='red')
34    plt.xlabel('Datetime')
35    plt.ylabel('Energy Production')
36    plt.title('Simulated Energy Production Over Time (Original vs Imputed)')
37    plt.legend()
38    plt.grid(True)
39    plt.show()
40
41# Main script
42if __name__ == "__main__":
43
44    start_date = datetime(2024, 1, 1)
45    end_date = datetime(2024, 4, 30)
46
47    # Generate the simulated dataset
48    energy_data_with_missing = simulate_energy_data(start_date, end_date)
49
50
51    plot_origin_data(energy_data_with_missing)
52
53
54    # Impute the missing values
55    energy_data_imputed = impute_missing_values(energy_data_with_missing.copy())
56
57    # Display the first few rows of the dataset
58    print(energy_data_imputed.head())
59
60    plot_full_data(energy_data_imputed,energy_data_with_missing)

 1
 2# Statistical comparison function using describe
 3def statistical_comparison(original_data, imputed_data):
 4    original_stats = original_data['Energy_Production'].describe()
 5    imputed_stats = imputed_data['Energy_Production'].describe()
 6    comparison = pd.DataFrame({
 7        'Metric': original_stats.index,
 8        'Original Data': original_stats.values,
 9        'Imputed Data': imputed_stats.values
10    })
11    return comparison
12
13comparison = statistical_comparison(energy_data_with_missing.dropna(), energy_data_imputed)
14print(comparison)

1  Metric  Original Data  Imputed Data
20  count   15553.000000  17281.000000
31   mean     185.155737    185.170568
42    std     127.062030    120.682665
53    min     -16.984058    -16.984058
64    25%      51.260972     53.345570
75    50%     257.313693    185.457881
86    75%     302.393653    298.754461
97    max     415.581945    415.581945

1  Metric  Original Data  Imputed Data Linear Regression
20  count   15553.000000  17281.000000
31   mean     185.155737    185.155865
42    std     127.062030    120.541640
53    min     -16.984058    -16.984058
64    25%      51.260972     53.436820
75    50%     257.313693    185.404682
86    75%     302.393653    298.653209
97    max     415.581945    415.581945

1
2
3# Autocorrelation analysis function
4def autocorrelation_analysis(original_data, imputed_data):
5    fig, axes = plt.subplots(1, 2, figsize=(15, 6))
6    plot_acf(original_data['Energy_Production'].dropna(), ax=axes[0], title='ACF of Original Data')
7    plot_acf(imputed_data['Energy_Production'], ax=axes[1], title='ACF of Imputed Data')
8    plt.tight_layout()
9    plt.show()

 1
 2# STL decomposition function to extract and plot trend component
 3def stl_decomposition_trend(original_data, imputed_data, period):
 4    stl_original = STL(original_data['Energy_Production'] , period=period)
 5    result_original = stl_original.fit()
 6    trend_original = result_original.trend
 7
 8    stl_imputed = STL(imputed_data['Energy_Production'], period=period)
 9    result_imputed = stl_imputed.fit()
10    trend_imputed = result_imputed.trend
11
12    plt.figure(figsize=(24, 6))
13    plt.plot(original_data['Datetime'], trend_original, label='Trend of Original Data', color='red')
14    plt.plot(imputed_data['Datetime'], trend_imputed, label='Trend of Imputed Data', color='blue')
15    plt.xlabel('Datetime')
16    plt.ylabel('Trend')
17    plt.title('STL Decomposition Trend of Original and Imputed Data')
18    plt.legend()
19    plt.grid(True)
20    plt.show()
21
22stl_decomposition_trend(energy_data_with_missing.dropna(), energy_data_imputed, period=144)  # Daily seasonality (144 10-min intervals in a day)

 1
 2
 3def stl_decomposition_seasonality(original_data, imputed_data, period):
 4    stl_original = STL(original_data['Energy_Production'], period=period)
 5    result_original = stl_original.fit()
 6    seasonality_original = result_original.seasonal
 7
 8    stl_imputed = STL(imputed_data['Energy_Production'], period=period)
 9    result_imputed = stl_imputed.fit()
10    seasonality_imputed = result_imputed.seasonal
11
12    plt.figure(figsize=(24, 6))
13    plt.plot(original_data['Datetime'], seasonality_original, label='Seasonality of Original Data', color='red')
14    plt.plot(imputed_data['Datetime'], seasonality_imputed, label='Seasonality of Imputed Data', color='blue')
15    plt.xlabel('Datetime')
16    plt.ylabel('Seasonality')
17    plt.title('STL Decomposition Seasonality of Original and Imputed Data')
18    plt.legend()
19    plt.grid(True)
20    plt.show()
21stl_decomposition_seasonality(energy_data_with_missing.dropna(), energy_data_imputed, period=144)  # Daily seasonality (144 10-min intervals in a day)