import pandas as pd
import numpy as np

# 讀取數(shù)據(jù)
df = pd.read_csv('data.csv')
print(df.head())

# 查看基本信息
print(df.info())
# 查看描述性統(tǒng)計信息
print(df.describe())
# 檢查缺失值
print(df.isnull().sum())

# 刪除含有缺失值的行
df = df.dropna()
# 刪除含有缺失值的列
df = df.dropna(axis=1)
填充缺失值
# 用均值填充數(shù)值列的缺失值
df['Age'] = df['Age'].fillna(df['Age'].mean())
# 用眾數(shù)填充分類列的缺失值
df['Gender'] = df['Gender'].fillna(df['Gender'].mode()[0])
# 用特定值填充
df['Income'] = df['Income'].fillna(0)
# 使用前一個值填充
df['Salary'] = df['Salary'].fillna(method='ffill')
# 使用后一個值填充
df['Salary'] = df['Salary'].fillna(method='bfill')
# 使用插值方法填充
df['Temperature'] = df['Temperature'].interpolate()

# 移除年齡大于100歲的記錄
df = df[df['Age'] <= 100]
使用 Z-score 方法
from scipy import stats
# 計算 Z-score
z_scores = np.abs(stats.zscore(df.select_dtypes(include=[np.number])))
# 移除 Z-score 大于 3 的記錄
df = df[(z_scores < 3).all(axis=1)]
使用 IQR 方法
Q1 = df.quantile(0.25)
Q3 = df.quantile(0.75)
IQR = Q3 - Q1
# 移除 IQR 范圍外的記錄
df = df[~((df < (Q1 - 1.5 * IQR)) | (df > (Q3 + 1.5 * IQR))).any(axis=1)]

# 檢查并刪除重復(fù)記錄
df = df.drop_duplicates()

# 將字符串轉(zhuǎn)換為日期
df['Date'] = pd.to_datetime(df['Date'])
# 將對象類型轉(zhuǎn)換為數(shù)值類型
df['Age'] = pd.to_numeric(df['Age'], errors='coerce')
# 將數(shù)值類型轉(zhuǎn)換為類別類型
df['Category'] = df['Category'].astype('category')

# 將所有文本轉(zhuǎn)換為小寫
df['Name'] = df['Name'].str.lower()
# 去除空格
df['Name'] = df['Name'].str.strip()
替換特定值
# 替換特定值
df['City'] = df['City'].replace({'New York City': 'New York', 'LA': 'Los Angeles'})

# 去除特殊字符
df['Comment'] = df['Comment'].str.replace('[^\w\s]', '', regex=True)

# 提取年份、月份、日
df['Year'] = df['Date'].dt.year
df['Month'] = df['Date'].dt.month
df['Day'] = df['Date'].dt.day

# 保存清洗后的數(shù)據(jù)
df.to_csv('cleaned_data.csv', index=False)

# 查看基本信息
print(df.info())
# 查看描述性統(tǒng)計信息
print(df.describe())
# 檢查缺失值
print(df.isnull().sum())
# 可視化數(shù)據(jù)分布
import matplotlib.pyplot as plt
df['Age'].hist(bins=20)
plt.show()

# 記錄每一步操作
with open('data_cleaning_log.txt', 'w') as f:
    f.write("Data Cleaning Log:\n")
    f.write(f"Initial shape: {df.shape}\n")
    # 示例：處理缺失值
    df = df.dropna()
    f.write(f"After dropping missing values: {df.shape}\n")
    # 示例：處理異常值
    df = df[df['Age'] <= 100]
    f.write(f"After removing outliers: {df.shape}\n")
    # 其他步驟...

# 第一階段：處理缺失值
df = df.dropna()
# 驗證結(jié)果
print(df.isnull().sum())
# 第二階段：處理異常值
df = df[df['Age'] <= 100]
# 驗證結(jié)果
print(df['Age'].describe())

# 斷言
assert df.isnull().sum().sum() == 0, "There are still missing values in the dataset"
assert (df['Age'] > 100).sum() == 0, "There are still age values greater than 100"
# 單元測試
import unittest
class TestDataCleaning(unittest.TestCase):
    def test_missing_values(self):
        self.assertEqual(df.isnull().sum().sum(), 0)
    def test_outliers(self):
        self.assertEqual((df['Age'] > 100).sum(), 0)
if __name__ == '__main__':
    unittest.main(argv=['first-arg-is-ignored'], exit=False)

# 定期復(fù)查數(shù)據(jù)
def check_data_quality(df):
    print("Checking data quality...")
    print("Missing values:", df.isnull().sum())
    print("Outliers in Age:", (df['Age'] > 100).sum())
check_data_quality(df)

# 初始化 Git 倉庫
git init
# 添加文件
git add data_cleaning_script.py
git add data_cleaning_log.txt
# 提交更改
git commit -m "Initial data cleaning script and log"