import pandas as pd

# DataFrame基本操作
df = pd.read_csv('data.csv')
df.head()
df.describe()

# 數(shù)據(jù)清洗
df.dropna()
df.fillna(0)
df['column'].astype('int64')

# 分組聚合
grouped = df.groupby('category')['value'].sum()

# 時(shí)間序列處理
df['date'] = pd.to_datetime(df['date'])
df.resample('D').sum()

# 合并操作
merged = pd.merge(df1, df2, on='key')

import numpy as np

# 數(shù)組創(chuàng)建與操作
arr = np.array([1, 2, 3, 4, 5])
matrix = np.arange(12).reshape(3, 4)

# 向量化運(yùn)算
result = arr * 2 + 1

# 矩陣運(yùn)算
a = np.random.rand(100, 50)
b = np.random.rand(50, 80)
c = np.dot(a, b)

# 統(tǒng)計(jì)函數(shù)
mean = np.mean(arr)
std = np.std(arr)
percentile = np.percentile(arr, 95)

# 傅里葉變換
from numpy.fft import fft
spectrum = fft(signal)

import polars as pl

# 創(chuàng)建DataFrame
df = pl.DataFrame({
    'name': ['Alice', 'Bob', 'Charlie'],
    'age': [25, 30, 28]
})

# 鏈?zhǔn)讲僮鳎ū磉_(dá)性強(qiáng)）
result = df.select([
    pl.col('name'),
    pl.col('age').apply(lambda x: x + 1).alias('age_plus_1')
]).filter(pl.col('age') > 26)

# 延遲計(jì)算（性能更優(yōu)）
lazy_df = pl.scan_csv('large_file.csv')
result = lazy_df.filter(pl.col('value') > 100).select(['name', 'value']).collect()

# 分組聚合
grouped = df.groupby('category').agg([
    pl.col('value').sum().alias('total'),
    pl.col('value').mean().alias('average')
])

# 與Pandas互轉(zhuǎn)
pd_df = df.to_pandas()
pl_df = pl.from_pandas(pd_df)

import duckdb

# SQL查詢CSV
result = duckdb.query("SELECT * FROM 'data.csv' WHERE age > 25")
df = result.to_df()

# 查詢Pandas DataFrame
pd_df = pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})
result = duckdb.query("SELECT * FROM pd_df WHERE age > 25")

# 復(fù)雜SQL操作
query = """
    SELECT 
        category,
        COUNT(*) as count,
        AVG(value) as avg_value,
        MAX(value) as max_value
    FROM 'sales.parquet'
    WHERE date >= '2024-01-01'
    GROUP BY category
    ORDER BY count DESC
"""
result = duckdb.query(query)

# 支持多表聯(lián)接
result = duckdb.query("""
    SELECT a.*, b.description
    FROM 'data1.csv' a
    JOIN 'data2.csv' b ON a.id = b.id
""")

from pyspark.sql import SparkSession

spark = SparkSession.builder.appName("data_analysis").getOrCreate()

# 從各種源讀取數(shù)據(jù)
df = spark.read.csv('large_data.csv', header=True, inferSchema=True)

# DataFrame操作
filtered = df.filter(df.age > 25)
grouped = df.groupBy('category').agg({'value': 'sum'})

# SQL查詢
df.createOrReplaceTempView('data')
result = spark.sql("SELECT category, COUNT(*) FROM data GROUP BY category")

# 機(jī)器學(xué)習(xí)集成
from pyspark.ml import Pipeline
from pyspark.ml.feature import StringIndexer
from pyspark.ml.classification import RandomForestClassifier

indexer = StringIndexer(inputCol='category', outputCol='categoryIndex')
rf = RandomForestClassifier(featuresCol='features', labelCol='label')
pipeline = Pipeline(stages=[indexer, rf])

# 混合使用多個(gè)庫的最佳實(shí)踐

# 1. 快速數(shù)據(jù)探索：使用Pandas
df = pd.read_csv('data.csv')
print(df.describe())

# 2. 性能敏感的處理：切換到Polars
pl_df = pl.from_pandas(df)
result = pl_df.filter(pl.col('value') > threshold).collect()

# 3. 復(fù)雜SQL操作：使用DuckDB
complex_result = duckdb.query("SELECT * FROM pl_df WHERE ...").to_df()

# 4. 最后用NumPy進(jìn)行數(shù)學(xué)計(jì)算
values = np.array(complex_result['value'])
stats = np.percentile(values, [25, 50, 75])