干货贴《机器学习5折保形回归》

策略作者: 水滴

```python
# 风险及免责提示：该策略由聚宽用户在聚宽社区分享，仅供学习交流使用。
# 原文一般包含策略说明，如有疑问请到原文和作者交流讨论。
# 原文网址：https://www.joinquant.com/view/community/detail/48114
# 标题：干货贴《机器学习5折保形回归》
# 作者：MarioC

from jqdata import *
from jqfactor import *
import numpy as np
import pandas as pd
import sklearn
from six import StringIO,BytesIO # py3的环境，使用BytesIO
from xgboost import XGBClassifier ,XGBRegressor
from sklearn.model_selection import KFold
# 初始化函数
def initialize(context):
    # 设定基准
    set_benchmark('000985.XSHG')
    # 用真实价格交易
    set_option('use_real_price', True)
    # 打开防未来函数
    set_option("avoid_future_data", True)
    # 将滑点设置为0
    set_slippage(FixedSlippage(0))
    # 设置交易成本万分之三，不同滑点影响可在归因分析中查看
    set_order_cost(OrderCost(open_tax=0, close_tax=0.001, open_commission=0.0003, close_commission=0.0003,
                             close_today_commission=0, min_commission=5), type='stock')
    # 过滤order中低于error级别的日志
    log.set_level('order', 'error')
    # 初始化全局变量
    g.no_trading_today_signal = False
    g.stock_num = 5
    g.hold_list = []  # 当前持仓的全部股票
    g.yesterday_HL_list = []  # 记录持仓中昨日涨停的股票

df1 = pd.read_csv(BytesIO(read_file('train_conformal_base.csv'))).dropna()
    df2 = pd.read_csv(BytesIO(read_file('test_conformal_base.csv'))).dropna()
    X_train = pd.concat([df1, df2], axis=0)
    
    y = X_train['LABEL']
    g.factor_list = ['non_linear_size',#非线性市值(风格因子)
                'beta',            #贝塔
                 'book_to_price_ratio',  #市面账值比
                'earnings_yield',  #盈利能力，
               'growth'            #成长 
               ]
    X = X_train[g.factor_list]
    
    
    # 定义五折交叉验证
    kf = KFold(n_splits=5, shuffle=False, random_state=42)
    
    # 遍历每一折
    for fold, (train_index, valid_index) in enumerate(kf.split(X, y)):
        print(f"Fold {fold+1}:")
        X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
        y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
        
        # 建立模型和保形回归器  
        model = XGBRegressor()  # 建立预测模型
        nc = NcFactory.create_nc(model)  # 建立不符合度量
        icp = IcpRegressor(nc)  # 建立一个归纳保形回归器
        
        # 拟合和校准模型  
        icp.fit(X_train.values, y_train.values)
        icp.calibrate(X_valid.values, y_valid.values)
        
        # 将模型保存到相应的属性中
        setattr(g, f"models{fold+1}", icp)

# 设置交易运行时间
    run_daily(prepare_stock_list, '9:05')
    run_monthly(weekly_adjustment, 1, '9:30')
    run_daily(check_limit_up, '14:00') 
    run_daily(close_account, '14:30')

# 1-1 准备股票池
def prepare_stock_list(context):
    # 获取已持有列表
    g.hold_list = []
    for position in list(context.portfolio.positions.values()):
        stock = position.security
        g.hold_list.append(stock)
    # 获取昨日涨停列表
    if g.hold_list != []:
        df = get_price(g.hold_list, end_date=context.previous_date, frequency='daily', fields=['close', 'high_limit'],
                       count=1, panel=False, fill_paused=False)
        df = df[df['close'] == df['high_limit']]
        g.yesterday_HL_list = list(df.code)
    else:
        g.yesterday_HL_list = []
        
    
# 1-2 选股模块
def get_stock_list(context):
    # 指定日期防止未来数据
    yesterday = context.previous_date
    today = context.current_dt
    # 获取初始列表
    stocks = get_all_securities('stock', yesterday).index.tolist()
    initial_list = filter_kcbj_stock(stocks)
    initial_list = filter_st_stock(initial_list)
    initial_list = filter_paused_stock(initial_list)
    initial_list = filter_new_stock(context, initial_list)
    initial_list = filter_limitup_stock(context,initial_list)
    initial_list = filter_limitdown_stock(context,initial_list)
    factor_data = get_factor_values(initial_list, g.factor_list, end_date=yesterday, count=1)
    df_jq_factor_value = pd.DataFrame(index=initial_list, columns=g.factor_list)
    for factor in g.factor_list:
        df_jq_factor_value[factor] = list(factor_data[factor].T.iloc[:, 0])
    df_jq_factor_value = data_preprocessing(df_jq_factor_value, initial_list, industry_code, yesterday)
    
    X=df_jq_factor_value.values
    Y1 = g.models1.predict(X, significance=0.05)
    Y2 = g.models2.predict(X, significance=0.05)
    Y3 = g.models3.predict(X, significance=0.05)
    Y4 = g.models4.predict(X, significance=0.05)
    Y5 = g.models5.predict(X, significance=0.05)
    Y=Y1+Y2+Y3+Y4+Y5
    y_lower = Y[:, 0]  
    y_upper = Y[:, 1]  
    y_pred = (y_lower + y_upper) / 2

tar = y_pred
    df = df_jq_factor_value
    df['total_score'] = list(tar)
    df = df.sort_values(by=['total_score'], ascending=False)  # 分数越高即预测未来收益越高，排序默认降序
    lst = df.index.tolist()
    lst = lst[:min(g.stock_num, len(lst))]
    return lst

# 1-3 整体调整持仓
def weekly_adjustment(context):
    # 获取应买入列表
    target_list = get_stock_list(context)
    # 调仓卖出
    for stock in g.hold_list:
        if (stock not in target_list) and (stock not in g.yesterday_HL_list):
            position = context.portfolio.positions[stock]
            close_position(position)
    position_count = len(context.portfolio.positions)
    target_num = len(target_list)
    if target_num > position_count:
        value = context.portfolio.cash / (target_num - position_count)
        for stock in target_list:
            if stock not in list(context.portfolio.positions.keys()):
                if open_position(stock, value):
                    if len(context.portfolio.positions) == target_num:
                        break

# 1-4 调整昨日涨停股票
def check_limit_up(context):
    now_time = context.current_dt
    if g.yesterday_HL_list != []:
        # 对昨日涨停股票观察到尾盘如不涨停则提前卖出，如果涨停即使不在应买入列表仍暂时持有
        for stock in g.yesterday_HL_list:
            current_data = get_price(stock, end_date=now_time, frequency='1m', fields=['close', 'high_limit'],
                                     skip_paused=False, fq='pre', count=1, panel=False, fill_paused=True)
            if current_data.iloc[0, 0] < current_data.iloc[0, 1]:
                log.info("[%s]涨停打开，卖出" % (stock))
                position = context.portfolio.positions[stock]
                close_position(position)
            else:
                log.info("[%s]涨停，继续持有" % (stock))

# 3-1 交易模块-自定义下单
def order_target_value_(security, value):
    if value == 0:
        log.debug("Selling out %s" % (security))
    else:
        log.debug("Order %s to value %f" % (security, value))
    return order_target_value(security, value)

# 3-2 交易模块-开仓
def open_position(security, value):
    order = order_target_value_(security, value)
    if order != None and order.filled > 0:
        return True
    return False

# 3-3 交易模块-平仓
def close_position(position):
    security = position.security
    order = order_target_value_(security, 0)  # 可能会因停牌失败
    if order != None:
        if order.status == OrderStatus.held and order.filled == order.amount:
            return True
    return False

# 4-1 判断今天是否为账户资金再平衡的日期
def today_is_between(context, start_date, end_date):
    today = context.current_dt.strftime('%m-%d')
    if (start_date <= today) and (today <= end_date):
        return True
    else:
        return False

# 4-2 清仓后次日资金可转
def close_account(context):
    if g.no_trading_today_signal == True:
        if len(g.hold_list) != 0:
            for stock in g.hold_list:
                position = context.portfolio.positions[stock]
                close_position(position)
                log.info("卖出[%s]" % (stock))

# 2-1 过滤停牌股票
def filter_paused_stock(stock_list):
    current_data = get_current_data()
    return [stock for stock in stock_list if not current_data[stock].paused]

# 2-2 过滤ST及其他具有退市标签的股票
def filter_st_stock(stock_list):
    current_data = get_current_data()
    return [stock for stock in stock_list
            if not current_data[stock].is_st
            and 'ST' not in current_data[stock].name
            and '*' not in current_data[stock].name
            and '退' not in current_data[stock].name]

# 2-3 过滤科创北交股票
def filter_kcbj_stock(stock_list):
    for stock in stock_list[:]:
        if stock[0] == '4' or stock[0] == '8' or stock[:2] == '68' or stock[0] == '3':
            stock_list.remove(stock)
    return stock_list

# 2-4 过滤涨停的股票
def filter_limitup_stock(context, stock_list):
    last_prices = history(1, unit='1m', field='close', security_list=stock_list)
    current_data = get_current_data()
    return [stock for stock in stock_list if stock in context.portfolio.positions.keys()
            or last_prices[stock][-1] < current_data[stock].high_limit]

# 2-5 过滤跌停的股票
def filter_limitdown_stock(context, stock_list):
    last_prices = history(1, unit='1m', field='close', security_list=stock_list)
    current_data = get_current_data()
    return [stock for stock in stock_list if stock in context.portfolio.positions.keys()
            or last_prices[stock][-1] > current_data[stock].low_limit]

# 2-6 过滤次新股
def filter_new_stock(context, stock_list):
    yesterday = context.previous_date
    return [stock for stock in stock_list if
            not yesterday - get_security_info(stock).start_date < datetime.timedelta(days=375)]

def get_industry_name(i_Constituent_Stocks, value):
    return [k for k, v in i_Constituent_Stocks.items() if value in v]

#缺失值处理
def replace_nan_indu(factor_data,stockList,industry_code,date):
    #把nan用行业平均值代替，依然会有nan，此时用所有股票平均值代替
    i_Constituent_Stocks={}
    data_temp=pd.DataFrame(index=industry_code,columns=factor_data.columns)
    for i in industry_code:
        temp = get_industry_stocks(i, date)
        i_Constituent_Stocks[i] = list(set(temp).intersection(set(stockList)))
        data_temp.loc[i]=mean(factor_data.loc[i_Constituent_Stocks[i],:])
    for factor in data_temp.columns:
        #行业缺失值用所有行业平均值代替
        null_industry=list(data_temp.loc[pd.isnull(data_temp[factor]),factor].keys())
        for i in null_industry:
            data_temp.loc[i,factor]=mean(data_temp[factor])
        null_stock=list(factor_data.loc[pd.isnull(factor_data[factor]),factor].keys())
        for i in null_stock:
            industry=get_industry_name(i_Constituent_Stocks, i)
            if industry:
                factor_data.loc[i,factor]=data_temp.loc[industry[0],factor] 
            else:
                factor_data.loc[i,factor]=mean(factor_data[factor])
    return factor_data
industry_code = ['801010','801020','801030','801040','801050','801080','801110','801120','801130','801140','801150',\
                    '801160','801170','801180','801200','801210','801230','801710','801720','801730','801740','801750',\
                   '801760','801770','801780','801790','801880','801890']
def data_preprocessing(factor_data,stockList,industry_code,date):
    #去极值
    factor_data=winsorize_med(factor_data, scale=5, inf2nan=False,axis=0)
    #缺失值处理
    factor_data=replace_nan_indu(factor_data,stockList,industry_code,date)
    factor_data=standardlize(factor_data,axis=0)
    return factor_data

#定义base

"""
docstring
"""

# Authors: Henrik Linusson

import abc
import numpy as np

from sklearn.base import BaseEstimator

class RegressorMixin(object):
	def __init__(self):
		super(RegressorMixin, self).__init__()

@classmethod
	def get_problem_type(cls):
		return 'regression'

class ClassifierMixin(object):
	def __init__(self):
		super(ClassifierMixin, self).__init__()

@classmethod
	def get_problem_type(cls):
		return 'classification'

class BaseModelAdapter(BaseEstimator):
	__metaclass__ = abc.ABCMeta

def __init__(self, model, fit_params=None):
		super(BaseModelAdapter, self).__init__()

self.model = model
		self.last_x, self.last_y = None, None
		self.clean = False
		self.fit_params = {} if fit_params is None else fit_params

def fit(self, x, y):
		"""Fits the model.

Parameters
		----------
		x : numpy array of shape [n_samples, n_features]
			Inputs of examples for fitting the model.

y : numpy array of shape [n_samples]
			Outputs of examples for fitting the model.

Returns
		-------
		None
		"""

self.model.fit(x, y, **self.fit_params)
		self.clean = False

def predict(self, x):
		"""Returns the prediction made by the underlying model.

Parameters
		----------
		x : numpy array of shape [n_samples, n_features]
			Inputs of test examples.

Returns
		-------
		y : numpy array of shape [n_samples]
			Predicted outputs of test examples.
		"""
		if (
			not self.clean or
			self.last_x is None or
			self.last_y is None or
			not np.array_equal(self.last_x, x)
		):
			self.last_x = x
			self.last_y = self._underlying_predict(x)
			self.clean = True

return self.last_y.copy()

@abc.abstractmethod
	def _underlying_predict(self, x):
		"""Produces a prediction using the encapsulated model.

Parameters
		----------
		x : numpy array of shape [n_samples, n_features]
			Inputs of test examples.

Returns
		-------
		y : numpy array of shape [n_samples]
			Predicted outputs of test examples.
		"""
		pass

class ClassifierAdapter(BaseModelAdapter):
	def __init__(self, model, fit_params=None):
		super(ClassifierAdapter, self).__init__(model, fit_params)

def _underlying_predict(self, x):
		return self.model.predict_proba(x)

class RegressorAdapter(BaseModelAdapter):
	def __init__(self, model, fit_params=None):
		super(RegressorAdapter, self).__init__(model, fit_params)

def _underlying_predict(self, x):
		return self.model.predict(x)

class OobMixin(object):
	def __init__(self, model, fit_params=None):
		super(OobMixin, self).__init__(model, fit_params)
		self.train_x = None

def fit(self, x, y):
		super(OobMixin, self).fit(x, y)
		self.train_x = x

def _underlying_predict(self, x):
		# TODO: sub-sampling of ensemble for test patterns
		oob = x == self.train_x

if hasattr(oob, 'all'):
			oob = oob.all()

if oob:
			return self._oob_prediction()
		else:
			return super(OobMixin, self)._underlying_predict(x)

class OobClassifierAdapter(OobMixin, ClassifierAdapter):
	def __init__(self, model, fit_params=None):
		super(OobClassifierAdapter, self).__init__(model, fit_params)

def _oob_prediction(self):
		return self.model.oob_decision_function_

class OobRegressorAdapter(OobMixin, RegressorAdapter):
	def __init__(self, model, fit_params=None):
		super(OobRegressorAdapter, self).__init__(model, fit_params)

def _oob_prediction(self):
		return self.model.oob_prediction_
		
#定义BaseScorer
class BaseScorer(sklearn.base.BaseEstimator):
	__metaclass__ = abc.ABCMeta

def __init__(self):
		super(BaseScorer, self).__init__()

@abc.abstractmethod
	def fit(self, x, y):
		pass

@abc.abstractmethod
	def score(self, x, y=None):
		pass

class RegressorNormalizer(BaseScorer):
	def __init__(self, base_model, normalizer_model, err_func):
		super(RegressorNormalizer, self).__init__()
		self.base_model = base_model
		self.normalizer_model = normalizer_model
		self.err_func = err_func

def fit(self, x, y):
		residual_prediction = self.base_model.predict(x)
		residual_error = np.abs(self.err_func.apply(residual_prediction, y))
		residual_error += 0.00001 # Add small term to avoid log(0)
		log_err = np.log(residual_error)
		self.normalizer_model.fit(x, log_err)

def score(self, x, y=None):
		norm = np.exp(self.normalizer_model.predict(x))
		return norm
#定义BaseModelNc
class BaseModelNc(BaseScorer):
	"""Base class for nonconformity scorers based on an underlying model.

Parameters
	----------
	model : ClassifierAdapter or RegressorAdapter
		Underlying classification model used for calculating nonconformity
		scores.

err_func : ClassificationErrFunc or RegressionErrFunc
		Error function object.

normalizer : BaseScorer
		Normalization model.

beta : float
		Normalization smoothing parameter. As the beta-value increases,
		the normalized nonconformity function approaches a non-normalized
		equivalent.
	"""
	def __init__(self, model, err_func, normalizer=None, beta=0):
		super(BaseModelNc, self).__init__()
		self.err_func = err_func
		self.model = model
		self.normalizer = normalizer
		self.beta = beta

# If we use sklearn.base.clone (e.g., during cross-validation),
		# object references get jumbled, so we need to make sure that the
		# normalizer has a reference to the proper model adapter, if applicable.
		if (self.normalizer is not None and
			hasattr(self.normalizer, 'base_model')):
			self.normalizer.base_model = self.model

self.last_x, self.last_y = None, None
		self.last_prediction = None
		self.clean = False

def fit(self, x, y):
		"""Fits the underlying model of the nonconformity scorer.

Parameters
		----------
		x : numpy array of shape [n_samples, n_features]
			Inputs of examples for fitting the underlying model.

y : numpy array of shape [n_samples]
			Outputs of examples for fitting the underlying model.

Returns
		-------
		None
		"""
		self.model.fit(x, y)
		if self.normalizer is not None:
			self.normalizer.fit(x, y)
		self.clean = False

def score(self, x, y=None):
		"""Calculates the nonconformity score of a set of samples.

Parameters
		----------
		x : numpy array of shape [n_samples, n_features]
			Inputs of examples for which to calculate a nonconformity score.

y : numpy array of shape [n_samples]
			Outputs of examples for which to calculate a nonconformity score.

Returns
		-------
		nc : numpy array of shape [n_samples]
			Nonconformity scores of samples.
		"""
		prediction = self.model.predict(x)
		n_test = x.shape[0]
		if self.normalizer is not None:
			norm = self.normalizer.score(x) + self.beta
		else:
			norm = np.ones(n_test)

return self.err_func.apply(prediction, y) / norm

#定义TcpClassifier
class TcpClassifier(BaseEstimator, ClassifierMixin):
	"""Transductive conformal classifier.

Parameters
	----------
	nc_function : BaseScorer
		Nonconformity scorer object used to calculate nonconformity of
		calibration examples and test patterns. Should implement ``fit(x, y)``
		and ``calc_nc(x, y)``.

smoothing : boolean
		Decides whether to use stochastic smoothing of p-values.

Attributes
	----------
	train_x : numpy array of shape [n_cal_examples, n_features]
		Inputs of training set.

train_y : numpy array of shape [n_cal_examples]
		Outputs of calibration set.

nc_function : BaseScorer
		Nonconformity scorer object used to calculate nonconformity scores.

classes : numpy array of shape [n_classes]
		List of class labels, with indices corresponding to output columns
		 of TcpClassifier.predict()

水滴

注册时间: