大數(shù)據(jù)預(yù)處理代碼

1、第一個(gè)例子：import matplotlib.pyplot as pltimport numpy as npfrom sklearn.model_selection import train_test_splitfrom sklearn.decomposition import PCAfrom sklearn.pipeline import make_pipelinefrom sklearn.preprocessing import

2、FunctionTransformerdef _generate_vector(shift=0.5, noise=15):return np.arange(1000) + (np.random.rand(1000) - shift) * noisedef generate_dataset():“““This dataset is two lines with a slope ~ 1, where one hasa y offset of

3、 ~100“““return np.vstack((np.vstack((_generate_vector(),_generate_vector() + 100,)).T,np.vstack((_generate_vector(),_generate_vector(),)).T,)), np.hstack((np.zeros(1000), np.ones(1000)))def all_but_first_column(X):return

4、 X[:, 1:]def drop_first_component(X, y):“““Create a pipeline with PCA and the column selector and use it totransform the dataset.“““pipeline = make_pipeline(PCA(), FunctionTransformer(all_but_first_column),)X_train, X_te

5、st, y_train, y_test = train_test_split(X, y)pipeline.fit(X_train, y_train)return pipeline.transform(X_test), y_testif __name__ == '__main__':第二個(gè)例子：from __future__ import print_functionprint(__doc__)# Code source:

6、 Thomas Unterthiner# License: BSD 3 clauseimport matplotlib.pyplot as pltimport numpy as npfrom sklearn.preprocessing import StandardScaler, RobustScaler# Create training and test datanp.random.seed(42)n_datapoints = 100

7、Cov = [[0.9, 0.0], [0.0, 20.0]]mu1 = [100.0, -3.0]mu2 = [101.0, -3.0]X1 = np.random.multivariate_normal(mean=mu1, cov=Cov, size=n_datapoints)X2 = np.random.multivariate_normal(mean=mu2, cov=Cov, size=n_datapoints)Y_train

8、 = np.hstack([[-1]*n_datapoints, [1]*n_datapoints])X_train = np.vstack([X1, X2])X1 = np.random.multivariate_normal(mean=mu1, cov=Cov, size=n_datapoints)X2 = np.random.multivariate_normal(mean=mu2, cov=Cov, size=n_datapoi

9、nts)Y_test = np.hstack([[-1]*n_datapoints, [1]*n_datapoints])X_test = np.vstack([X1, X2])X_train[0, 0] = -1000 # a fairly large outlier# Scale datastandard_scaler = StandardScaler()Xtr_s = standard_scaler.fit_transform(X

10、_train)Xte_s = standard_scaler.transform(X_test)robust_scaler = RobustScaler()Xtr_r = robust_scaler.fit_transform(X_train)Xte_r = robust_scaler.transform(X_test)# Plot datafig, ax = plt.subplots(1, 3, figsize=(12, 4))ax[