Custom Metrics in PyCaret for Numerai
Numerai is a data science competition that uses non-standard metrics for scoring. This post shows how to use PyCaret for AutoML with custom metrics added to it for optimization.
• Justin Malloy • 14 min read
numerai pycaret scikit-learn datascience
import numpy as np
import pandas as pd
import csv
from pathlib import Path
from sklearn.metrics import make_scorer
from scipy.stats import spearmanr, pearsonr
from pycaret.regression import *
TARGET_NAME = f"target"
PREDICTION_NAME = f"prediction"
training_data = read_csv("numerai_training_data.csv")
tournament_data = read_csv("numerai_tournament_data.csv")
# Sampling for ease of demo/repeatability - obviously would use full set for competition
training_data = training_data.sample(500)
tournament_data = tournament_data.sample(500)
feature_names = [f for f in training_data.columns if f.startswith("feature") or f == TARGET_NAME]
training_data.head(5)
| era | data_type | feature_intelligence1 | feature_intelligence2 | feature_intelligence3 | feature_intelligence4 | feature_intelligence5 | feature_intelligence6 | feature_intelligence7 | feature_intelligence8 | feature_intelligence9 | feature_intelligence10 | feature_intelligence11 | feature_intelligence12 | feature_charisma1 | feature_charisma2 | feature_charisma3 | feature_charisma4 | feature_charisma5 | feature_charisma6 | feature_charisma7 | feature_charisma8 | feature_charisma9 | feature_charisma10 | feature_charisma11 | feature_charisma12 | feature_charisma13 | feature_charisma14 | feature_charisma15 | feature_charisma16 | feature_charisma17 | feature_charisma18 | feature_charisma19 | feature_charisma20 | feature_charisma21 | feature_charisma22 | feature_charisma23 | feature_charisma24 | feature_charisma25 | feature_charisma26 | feature_charisma27 | feature_charisma28 | feature_charisma29 | feature_charisma30 | feature_charisma31 | feature_charisma32 | feature_charisma33 | feature_charisma34 | feature_charisma35 | feature_charisma36 | feature_charisma37 | feature_charisma38 | feature_charisma39 | feature_charisma40 | feature_charisma41 | feature_charisma42 | feature_charisma43 | feature_charisma44 | feature_charisma45 | feature_charisma46 | feature_charisma47 | feature_charisma48 | feature_charisma49 | feature_charisma50 | feature_charisma51 | feature_charisma52 | feature_charisma53 | feature_charisma54 | feature_charisma55 | feature_charisma56 | feature_charisma57 | feature_charisma58 | feature_charisma59 | feature_charisma60 | feature_charisma61 | feature_charisma62 | feature_charisma63 | feature_charisma64 | feature_charisma65 | feature_charisma66 | feature_charisma67 | feature_charisma68 | feature_charisma69 | feature_charisma70 | feature_charisma71 | feature_charisma72 | feature_charisma73 | feature_charisma74 | feature_charisma75 | feature_charisma76 | feature_charisma77 | feature_charisma78 | feature_charisma79 | feature_charisma80 | feature_charisma81 | feature_charisma82 | feature_charisma83 | feature_charisma84 | feature_charisma85 | feature_charisma86 | feature_strength1 | feature_strength2 | feature_strength3 | feature_strength4 | feature_strength5 | feature_strength6 | feature_strength7 | feature_strength8 | feature_strength9 | feature_strength10 | feature_strength11 | feature_strength12 | feature_strength13 | feature_strength14 | feature_strength15 | feature_strength16 | feature_strength17 | feature_strength18 | feature_strength19 | feature_strength20 | feature_strength21 | feature_strength22 | feature_strength23 | feature_strength24 | feature_strength25 | feature_strength26 | feature_strength27 | feature_strength28 | feature_strength29 | feature_strength30 | feature_strength31 | feature_strength32 | feature_strength33 | feature_strength34 | feature_strength35 | feature_strength36 | feature_strength37 | feature_strength38 | feature_dexterity1 | feature_dexterity2 | feature_dexterity3 | feature_dexterity4 | feature_dexterity5 | feature_dexterity6 | feature_dexterity7 | feature_dexterity8 | feature_dexterity9 | feature_dexterity10 | feature_dexterity11 | feature_dexterity12 | feature_dexterity13 | feature_dexterity14 | feature_constitution1 | feature_constitution2 | feature_constitution3 | feature_constitution4 | feature_constitution5 | feature_constitution6 | feature_constitution7 | feature_constitution8 | feature_constitution9 | feature_constitution10 | feature_constitution11 | feature_constitution12 | feature_constitution13 | feature_constitution14 | feature_constitution15 | feature_constitution16 | feature_constitution17 | feature_constitution18 | feature_constitution19 | feature_constitution20 | feature_constitution21 | feature_constitution22 | feature_constitution23 | feature_constitution24 | feature_constitution25 | feature_constitution26 | feature_constitution27 | feature_constitution28 | feature_constitution29 | feature_constitution30 | feature_constitution31 | feature_constitution32 | feature_constitution33 | feature_constitution34 | feature_constitution35 | feature_constitution36 | feature_constitution37 | feature_constitution38 | feature_constitution39 | feature_constitution40 | feature_constitution41 | feature_constitution42 | feature_constitution43 | feature_constitution44 | feature_constitution45 | feature_constitution46 | feature_constitution47 | feature_constitution48 | feature_constitution49 | feature_constitution50 | feature_constitution51 | feature_constitution52 | feature_constitution53 | feature_constitution54 | feature_constitution55 | feature_constitution56 | feature_constitution57 | feature_constitution58 | feature_constitution59 | feature_constitution60 | feature_constitution61 | feature_constitution62 | feature_constitution63 | feature_constitution64 | feature_constitution65 | feature_constitution66 | feature_constitution67 | feature_constitution68 | feature_constitution69 | feature_constitution70 | feature_constitution71 | feature_constitution72 | feature_constitution73 | feature_constitution74 | feature_constitution75 | feature_constitution76 | feature_constitution77 | feature_constitution78 | feature_constitution79 | feature_constitution80 | feature_constitution81 | feature_constitution82 | feature_constitution83 | feature_constitution84 | feature_constitution85 | feature_constitution86 | feature_constitution87 | feature_constitution88 | feature_constitution89 | feature_constitution90 | feature_constitution91 | feature_constitution92 | feature_constitution93 | feature_constitution94 | feature_constitution95 | feature_constitution96 | feature_constitution97 | feature_constitution98 | feature_constitution99 | feature_constitution100 | feature_constitution101 | feature_constitution102 | feature_constitution103 | feature_constitution104 | feature_constitution105 | feature_constitution106 | feature_constitution107 | feature_constitution108 | feature_constitution109 | feature_constitution110 | feature_constitution111 | feature_constitution112 | feature_constitution113 | feature_constitution114 | feature_wisdom1 | feature_wisdom2 | feature_wisdom3 | feature_wisdom4 | feature_wisdom5 | feature_wisdom6 | feature_wisdom7 | feature_wisdom8 | feature_wisdom9 | feature_wisdom10 | feature_wisdom11 | feature_wisdom12 | feature_wisdom13 | feature_wisdom14 | feature_wisdom15 | feature_wisdom16 | feature_wisdom17 | feature_wisdom18 | feature_wisdom19 | feature_wisdom20 | feature_wisdom21 | feature_wisdom22 | feature_wisdom23 | feature_wisdom24 | feature_wisdom25 | feature_wisdom26 | feature_wisdom27 | feature_wisdom28 | feature_wisdom29 | feature_wisdom30 | feature_wisdom31 | feature_wisdom32 | feature_wisdom33 | feature_wisdom34 | feature_wisdom35 | feature_wisdom36 | feature_wisdom37 | feature_wisdom38 | feature_wisdom39 | feature_wisdom40 | feature_wisdom41 | feature_wisdom42 | feature_wisdom43 | feature_wisdom44 | feature_wisdom45 | feature_wisdom46 | target | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| id | |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| n6346c8f8784a565 | era75 | train | 0.75 | 1.00 | 1.00 | 0.50 | 0.50 | 0.50 | 0.25 | 1.00 | 0.75 | 0.75 | 0.75 | 0.50 | 1.00 | 0.25 | 0.50 | 0.00 | 0.00 | 0.75 | 0.25 | 1.00 | 0.25 | 0.25 | 1.00 | 0.50 | 0.75 | 0.00 | 1.00 | 0.25 | 0.00 | 1.00 | 1.00 | 0.00 | 0.75 | 0.00 | 0.75 | 0.25 | 1.00 | 0.00 | 0.00 | 0.75 | 0.00 | 0.00 | 0.5 | 1.00 | 0.00 | 0.25 | 0.00 | 0.75 | 1.00 | 0.00 | 0.50 | 0.00 | 0.00 | 0.75 | 1.00 | 0.00 | 0.50 | 0.50 | 1.0 | 1.00 | 0.25 | 0.50 | 0.50 | 1.00 | 0.75 | 0.75 | 0.75 | 0.0 | 0.00 | 1.00 | 0.00 | 0.00 | 0.0 | 0.75 | 0.75 | 0.00 | 0.00 | 0.00 | 0.0 | 0.50 | 0.25 | 0.75 | 0.0 | 0.25 | 0.25 | 0.00 | 0.75 | 1.00 | 0.00 | 0.25 | 0.00 | 0.75 | 1.00 | 0.25 | 0.00 | 1.00 | 0.5 | 0.75 | 0.00 | 0.00 | 1.0 | 0.00 | 0.75 | 0.5 | 0.00 | 0.75 | 0.00 | 1.00 | 0.00 | 0.75 | 0.25 | 1.00 | 1.00 | 0.00 | 0.50 | 1.0 | 1.0 | 0.75 | 0.00 | 0.00 | 0.00 | 1.00 | 0.00 | 0.00 | 1.00 | 0.75 | 1.00 | 0.75 | 0.00 | 0.00 | 0.00 | 1.00 | 0.00 | 1.00 | 1.0 | 0.75 | 0.25 | 0.75 | 1.0 | 1.00 | 0.75 | 1.00 | 1.00 | 1.00 | 0.5 | 1.0 | 0.75 | 1.00 | 1.0 | 1.00 | 0.25 | 0.25 | 1.00 | 0.00 | 0.5 | 0.5 | 1.00 | 0.00 | 1.00 | 0.5 | 0.0 | 0.75 | 1.0 | 0.75 | 0.0 | 0.00 | 0.00 | 0.00 | 0.00 | 0.75 | 0.50 | 0.0 | 1.0 | 0.50 | 0.25 | 0.75 | 1.0 | 0.00 | 1.00 | 0.50 | 0.75 | 0.25 | 1.00 | 0.0 | 0.00 | 0.00 | 0.00 | 0.25 | 0.0 | 0.00 | 0.00 | 0.75 | 0.75 | 1.0 | 0.00 | 0.25 | 0.00 | 0.00 | 0.00 | 1.00 | 0.50 | 1.00 | 0.75 | 0.0 | 1.0 | 0.00 | 0.25 | 0.00 | 0.50 | 0.5 | 0.75 | 0.50 | 0.0 | 0.00 | 0.00 | 1.0 | 0.50 | 0.0 | 1.00 | 0.75 | 0.75 | 0.25 | 0.25 | 0.50 | 0.00 | 0.25 | 0.0 | 0.00 | 1.0 | 0.50 | 0.75 | 0.50 | 0.00 | 0.75 | 0.75 | 0.75 | 0.5 | 0.00 | 0.00 | 0.0 | 0.00 | 0.25 | 1.0 | 1.00 | 0.00 | 1.00 | 0.00 | 0.25 | 0.00 | 0.00 | 0.00 | 0.00 | 0.25 | 0.5 | 0.00 | 0.0 | 0.00 | 0.25 | 0.0 | 0.0 | 0.25 | 0.25 | 0.00 | 0.5 | 0.00 | 0.00 | 0.00 | 0.25 | 0.25 | 0.0 | 0.25 | 0.00 | 0.00 | 0.0 | 0.25 | 0.00 | 0.00 | 0.00 | 0.00 | 0.25 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.25 | 0.00 | 0.00 | 0.25 | 0.00 | 0.00 | 0.25 | 0.00 | 0.25 | 0.00 | 0.25 | 0.00 | 0.25 | 0.00 | 0.50 | 0.00 | 0.00 | 0.00 | 0.50 | 0.50 | 0.75 | 0.00 | 0.25 | 0.00 | 0.25 | 0.25 |
| n46dccf4bc8006a8 | era75 | train | 0.00 | 1.00 | 1.00 | 0.00 | 0.25 | 0.50 | 0.25 | 0.00 | 1.00 | 1.00 | 0.00 | 0.25 | 0.25 | 0.75 | 0.25 | 1.00 | 0.50 | 1.00 | 1.00 | 0.75 | 0.00 | 0.25 | 0.50 | 1.00 | 0.75 | 1.00 | 0.25 | 1.00 | 1.00 | 1.00 | 0.50 | 0.25 | 0.75 | 0.25 | 0.75 | 0.75 | 0.25 | 1.00 | 0.75 | 0.75 | 0.75 | 0.75 | 1.0 | 1.00 | 1.00 | 1.00 | 1.00 | 1.00 | 0.75 | 1.00 | 0.75 | 0.75 | 0.75 | 1.00 | 1.00 | 1.00 | 0.25 | 0.50 | 0.0 | 0.75 | 0.50 | 1.00 | 1.00 | 0.75 | 0.75 | 1.00 | 1.00 | 0.5 | 0.50 | 0.75 | 0.50 | 0.25 | 0.5 | 1.00 | 1.00 | 0.75 | 0.50 | 0.75 | 1.0 | 0.75 | 0.25 | 1.00 | 1.0 | 1.00 | 0.25 | 1.00 | 1.00 | 1.00 | 0.75 | 0.50 | 0.75 | 1.00 | 1.00 | 0.75 | 0.50 | 0.75 | 1.0 | 0.75 | 0.50 | 0.50 | 0.0 | 0.75 | 0.50 | 0.5 | 0.50 | 0.25 | 0.50 | 0.50 | 0.50 | 0.25 | 0.75 | 0.75 | 0.75 | 1.00 | 0.75 | 0.5 | 0.0 | 0.50 | 0.75 | 0.50 | 0.50 | 0.00 | 0.00 | 0.50 | 0.75 | 0.75 | 0.00 | 0.50 | 0.25 | 0.25 | 0.50 | 0.75 | 0.75 | 0.75 | 0.5 | 0.50 | 1.00 | 1.00 | 1.0 | 0.75 | 1.00 | 1.00 | 1.00 | 1.00 | 1.0 | 1.0 | 1.00 | 0.00 | 1.0 | 0.25 | 0.25 | 1.00 | 0.25 | 0.75 | 0.0 | 0.0 | 0.00 | 0.25 | 0.75 | 0.0 | 1.0 | 0.00 | 0.0 | 0.75 | 0.5 | 0.50 | 0.25 | 0.25 | 1.00 | 0.75 | 0.25 | 0.5 | 0.0 | 0.75 | 0.50 | 0.75 | 0.0 | 0.75 | 0.25 | 0.75 | 1.00 | 0.50 | 1.00 | 0.5 | 0.75 | 0.75 | 0.50 | 0.25 | 0.5 | 0.50 | 0.00 | 0.25 | 0.50 | 0.0 | 1.00 | 0.00 | 1.00 | 0.25 | 1.00 | 0.50 | 1.00 | 1.00 | 0.50 | 1.0 | 0.0 | 0.25 | 0.25 | 1.00 | 1.00 | 1.0 | 1.00 | 0.50 | 0.5 | 0.75 | 0.00 | 0.0 | 0.75 | 1.0 | 1.00 | 0.50 | 0.75 | 0.00 | 1.00 | 0.25 | 0.25 | 0.75 | 0.5 | 0.50 | 0.0 | 0.50 | 0.75 | 0.50 | 0.50 | 0.75 | 0.75 | 0.75 | 1.0 | 0.25 | 0.50 | 0.0 | 0.00 | 0.50 | 0.0 | 0.75 | 1.00 | 1.00 | 0.25 | 0.75 | 0.75 | 0.75 | 0.25 | 0.25 | 0.25 | 1.0 | 0.50 | 0.5 | 0.25 | 0.00 | 0.5 | 0.0 | 0.50 | 0.00 | 0.75 | 0.5 | 0.00 | 0.50 | 0.50 | 0.25 | 0.25 | 0.0 | 0.50 | 0.25 | 0.50 | 0.5 | 0.25 | 0.50 | 0.75 | 0.50 | 0.25 | 0.25 | 0.25 | 0.75 | 0.25 | 0.25 | 0.25 | 0.50 | 0.75 | 0.00 | 0.50 | 0.00 | 0.25 | 0.25 | 0.25 | 0.25 | 0.25 | 0.50 | 0.00 | 0.75 | 0.75 | 0.00 | 0.25 | 0.00 | 0.25 | 0.75 | 0.75 | 0.00 | 0.25 | 0.25 | 0.50 | 0.50 | 0.25 |
| n3e1605f37041c0c | era6 | train | 0.75 | 0.25 | 0.00 | 0.75 | 0.75 | 0.50 | 0.25 | 0.75 | 0.00 | 0.00 | 0.75 | 1.00 | 0.00 | 0.50 | 0.75 | 0.00 | 0.75 | 1.00 | 0.50 | 0.50 | 1.00 | 1.00 | 0.25 | 0.75 | 0.00 | 0.75 | 0.50 | 0.25 | 0.00 | 0.00 | 0.00 | 0.75 | 0.00 | 0.25 | 1.00 | 0.75 | 0.75 | 0.25 | 0.25 | 0.00 | 1.00 | 0.25 | 0.0 | 0.00 | 0.75 | 0.50 | 0.50 | 0.00 | 0.00 | 0.25 | 1.00 | 1.00 | 1.00 | 1.00 | 0.00 | 0.25 | 0.75 | 0.25 | 1.0 | 0.00 | 0.75 | 0.00 | 0.00 | 0.00 | 0.00 | 1.00 | 0.50 | 0.5 | 1.00 | 0.25 | 1.00 | 0.25 | 1.0 | 0.25 | 0.00 | 0.75 | 0.75 | 0.00 | 0.0 | 0.00 | 1.00 | 0.00 | 1.0 | 0.50 | 0.75 | 0.25 | 1.00 | 0.00 | 0.00 | 1.00 | 0.75 | 0.50 | 0.00 | 0.00 | 0.75 | 1.00 | 0.0 | 1.00 | 1.00 | 0.00 | 1.0 | 1.00 | 0.50 | 0.5 | 0.50 | 0.50 | 0.50 | 0.75 | 1.00 | 0.25 | 0.50 | 1.00 | 0.75 | 0.25 | 0.50 | 1.0 | 1.0 | 0.50 | 1.00 | 0.75 | 0.00 | 1.00 | 0.00 | 0.50 | 0.50 | 0.75 | 0.75 | 0.50 | 0.00 | 0.00 | 0.50 | 0.75 | 0.50 | 0.75 | 1.0 | 0.50 | 0.00 | 0.00 | 0.0 | 0.25 | 0.00 | 0.00 | 0.25 | 0.00 | 0.0 | 0.0 | 0.00 | 0.50 | 0.0 | 0.50 | 0.75 | 0.00 | 0.00 | 0.00 | 1.0 | 1.0 | 0.50 | 0.25 | 0.00 | 1.0 | 0.0 | 1.00 | 0.0 | 0.00 | 0.0 | 0.50 | 0.25 | 0.75 | 0.00 | 0.25 | 0.25 | 0.5 | 0.0 | 0.00 | 0.25 | 0.75 | 0.5 | 0.00 | 1.00 | 0.00 | 0.25 | 0.50 | 0.00 | 0.0 | 0.00 | 0.25 | 0.00 | 0.50 | 0.5 | 0.50 | 0.50 | 0.00 | 0.25 | 0.5 | 0.00 | 1.00 | 0.25 | 0.75 | 0.25 | 1.00 | 0.00 | 1.00 | 0.50 | 0.0 | 0.5 | 1.00 | 0.75 | 1.00 | 0.00 | 0.0 | 0.00 | 0.25 | 0.0 | 0.25 | 0.00 | 0.5 | 0.25 | 1.0 | 0.75 | 0.75 | 0.75 | 0.00 | 0.00 | 1.00 | 0.75 | 0.50 | 0.5 | 0.00 | 0.5 | 1.00 | 0.00 | 0.00 | 0.50 | 0.75 | 0.75 | 1.00 | 0.0 | 0.00 | 0.50 | 0.0 | 1.00 | 0.25 | 0.5 | 1.00 | 0.00 | 0.00 | 0.25 | 0.00 | 0.25 | 0.50 | 1.00 | 1.00 | 0.00 | 0.0 | 0.00 | 0.5 | 0.50 | 0.50 | 0.0 | 1.0 | 0.25 | 1.00 | 0.25 | 0.5 | 0.00 | 0.25 | 1.00 | 0.00 | 0.00 | 1.0 | 0.00 | 0.00 | 1.00 | 0.0 | 1.00 | 0.25 | 0.00 | 0.00 | 0.00 | 0.00 | 0.25 | 0.00 | 0.00 | 0.00 | 0.50 | 1.00 | 0.00 | 1.00 | 0.00 | 0.75 | 0.00 | 0.50 | 0.00 | 0.25 | 0.00 | 0.00 | 0.75 | 0.00 | 0.00 | 0.25 | 0.25 | 0.25 | 0.00 | 0.00 | 0.00 | 0.25 | 0.75 | 0.00 | 0.00 | 0.00 | 0.25 |
| na662ad0039c3091 | era107 | train | 0.50 | 0.75 | 0.75 | 0.25 | 0.50 | 0.75 | 0.75 | 0.00 | 0.75 | 0.75 | 0.25 | 0.50 | 0.25 | 0.25 | 1.00 | 0.75 | 0.25 | 1.00 | 1.00 | 0.75 | 0.50 | 1.00 | 0.75 | 0.50 | 0.50 | 0.50 | 0.75 | 0.50 | 0.75 | 0.25 | 0.75 | 0.25 | 1.00 | 0.50 | 0.50 | 0.25 | 0.25 | 0.25 | 0.50 | 0.25 | 0.50 | 0.25 | 0.5 | 0.25 | 0.50 | 1.00 | 0.50 | 1.00 | 0.25 | 0.25 | 0.50 | 0.50 | 0.25 | 0.50 | 1.00 | 0.50 | 1.00 | 1.00 | 0.0 | 0.00 | 1.00 | 0.25 | 0.25 | 0.50 | 0.25 | 0.75 | 1.00 | 1.0 | 0.25 | 0.75 | 0.25 | 0.75 | 0.5 | 0.75 | 1.00 | 0.25 | 1.00 | 1.00 | 0.5 | 0.50 | 0.75 | 1.00 | 0.5 | 1.00 | 0.75 | 0.50 | 0.50 | 0.25 | 0.25 | 1.00 | 0.25 | 0.25 | 0.25 | 0.75 | 0.00 | 0.25 | 1.0 | 0.25 | 0.75 | 0.50 | 0.5 | 0.75 | 1.00 | 0.5 | 0.25 | 0.50 | 0.25 | 0.75 | 0.25 | 0.50 | 1.00 | 1.00 | 0.50 | 0.25 | 0.50 | 1.0 | 0.5 | 0.50 | 0.25 | 0.25 | 0.25 | 0.75 | 0.25 | 0.75 | 0.50 | 0.25 | 0.75 | 0.50 | 0.00 | 0.25 | 0.75 | 1.00 | 0.25 | 0.50 | 1.0 | 0.50 | 0.75 | 0.75 | 0.5 | 0.00 | 0.50 | 0.25 | 0.00 | 0.50 | 0.5 | 0.5 | 0.50 | 0.00 | 0.5 | 0.25 | 0.50 | 0.75 | 0.00 | 0.00 | 0.0 | 0.0 | 0.75 | 0.25 | 0.75 | 0.0 | 0.5 | 0.50 | 0.5 | 0.25 | 0.5 | 0.25 | 0.00 | 0.00 | 0.50 | 1.00 | 0.00 | 0.0 | 0.0 | 0.75 | 0.00 | 1.00 | 1.0 | 0.00 | 0.50 | 0.25 | 0.50 | 0.25 | 0.75 | 0.0 | 0.25 | 0.25 | 0.75 | 0.75 | 0.0 | 0.25 | 0.25 | 0.25 | 1.00 | 1.0 | 0.75 | 0.00 | 0.50 | 0.00 | 0.25 | 0.25 | 0.25 | 0.75 | 0.75 | 1.0 | 1.0 | 0.25 | 0.25 | 1.00 | 0.25 | 1.0 | 1.00 | 1.00 | 0.5 | 0.25 | 0.25 | 1.0 | 0.25 | 1.0 | 1.00 | 0.00 | 1.00 | 0.00 | 0.50 | 0.00 | 0.25 | 0.25 | 0.0 | 0.25 | 0.5 | 0.75 | 0.25 | 0.75 | 0.25 | 1.00 | 1.00 | 1.00 | 1.0 | 0.00 | 0.00 | 0.0 | 0.25 | 1.00 | 1.0 | 1.00 | 1.00 | 0.75 | 0.00 | 0.25 | 1.00 | 0.25 | 0.50 | 0.25 | 0.25 | 0.5 | 0.25 | 0.5 | 0.00 | 0.25 | 0.0 | 0.5 | 0.25 | 0.00 | 0.00 | 1.0 | 0.75 | 0.75 | 0.75 | 1.00 | 1.00 | 0.5 | 1.00 | 0.00 | 1.00 | 0.0 | 0.25 | 0.75 | 1.00 | 0.75 | 0.25 | 0.75 | 1.00 | 0.75 | 0.00 | 1.00 | 0.50 | 1.00 | 0.50 | 0.75 | 0.50 | 0.25 | 0.50 | 0.00 | 0.75 | 1.00 | 0.50 | 0.25 | 1.00 | 1.00 | 1.00 | 0.75 | 0.75 | 0.75 | 0.50 | 0.75 | 1.00 | 0.25 | 1.00 | 0.50 | 0.75 | 1.00 | 0.50 |
| nf324eff3bcbf017 | era99 | train | 0.25 | 0.25 | 0.50 | 0.75 | 0.00 | 0.25 | 0.50 | 0.75 | 0.50 | 0.50 | 0.75 | 0.25 | 0.50 | 0.50 | 0.00 | 0.00 | 0.75 | 1.00 | 0.25 | 0.00 | 1.00 | 0.75 | 0.00 | 0.25 | 0.25 | 0.00 | 0.00 | 0.50 | 0.00 | 0.00 | 0.50 | 0.75 | 0.00 | 0.00 | 0.00 | 0.50 | 0.50 | 0.75 | 0.50 | 0.00 | 1.00 | 0.25 | 0.0 | 0.00 | 0.00 | 1.00 | 0.75 | 0.00 | 0.00 | 0.50 | 0.00 | 1.00 | 0.25 | 1.00 | 0.25 | 1.00 | 0.25 | 0.00 | 0.5 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 1.00 | 1.00 | 0.0 | 0.75 | 0.25 | 0.25 | 0.00 | 1.0 | 0.25 | 0.00 | 0.75 | 0.00 | 0.25 | 0.5 | 0.00 | 1.00 | 0.25 | 1.0 | 0.50 | 0.00 | 1.00 | 1.00 | 0.00 | 0.50 | 1.00 | 0.75 | 0.25 | 0.00 | 0.00 | 0.75 | 1.00 | 0.0 | 1.00 | 1.00 | 0.25 | 1.0 | 1.00 | 0.00 | 0.5 | 0.50 | 0.75 | 0.25 | 1.00 | 0.50 | 1.00 | 0.00 | 0.00 | 1.00 | 0.50 | 0.50 | 1.0 | 1.0 | 0.50 | 0.50 | 0.00 | 0.25 | 1.00 | 0.00 | 0.50 | 0.50 | 0.75 | 1.00 | 1.00 | 0.00 | 0.25 | 0.50 | 0.00 | 0.50 | 1.00 | 1.0 | 0.50 | 0.50 | 0.50 | 0.0 | 0.00 | 0.25 | 0.00 | 0.00 | 0.25 | 0.5 | 0.5 | 0.00 | 0.25 | 0.0 | 0.00 | 0.75 | 0.00 | 0.50 | 0.00 | 0.0 | 0.5 | 1.00 | 0.00 | 0.00 | 0.5 | 0.0 | 1.00 | 0.5 | 0.00 | 1.0 | 0.00 | 0.00 | 1.00 | 0.25 | 0.75 | 0.00 | 0.5 | 0.5 | 0.00 | 0.50 | 0.75 | 1.0 | 0.00 | 1.00 | 0.00 | 0.25 | 1.00 | 0.00 | 0.0 | 0.25 | 0.25 | 1.00 | 1.00 | 1.0 | 0.00 | 0.50 | 0.00 | 0.75 | 0.5 | 0.25 | 1.00 | 0.25 | 0.50 | 0.25 | 1.00 | 0.00 | 0.75 | 1.00 | 0.0 | 0.5 | 1.00 | 1.00 | 0.25 | 0.00 | 0.0 | 0.00 | 0.75 | 0.0 | 0.50 | 0.00 | 0.5 | 0.50 | 0.0 | 1.00 | 0.00 | 1.00 | 0.50 | 0.00 | 0.00 | 1.00 | 0.25 | 0.5 | 0.00 | 0.5 | 0.75 | 0.00 | 0.00 | 0.25 | 0.75 | 0.75 | 0.50 | 0.0 | 0.50 | 0.25 | 0.5 | 1.00 | 1.00 | 0.5 | 0.50 | 0.25 | 0.00 | 0.50 | 0.50 | 1.00 | 0.25 | 0.75 | 1.00 | 0.50 | 0.0 | 0.00 | 1.0 | 0.50 | 0.50 | 0.0 | 1.0 | 0.50 | 1.00 | 0.50 | 1.0 | 0.00 | 0.00 | 0.50 | 0.75 | 0.00 | 0.0 | 0.00 | 0.50 | 0.75 | 0.5 | 0.25 | 0.00 | 0.25 | 0.00 | 0.00 | 1.00 | 0.25 | 0.00 | 0.50 | 0.00 | 0.25 | 0.75 | 0.00 | 0.50 | 0.00 | 1.00 | 0.00 | 0.00 | 0.25 | 1.00 | 0.00 | 0.00 | 0.50 | 0.00 | 0.00 | 0.50 | 0.25 | 0.00 | 0.00 | 0.00 | 0.00 | 0.25 | 0.50 | 0.00 | 1.00 | 0.00 | 0.25 |
reg = setup(
data=training_data[feature_names],
target=TARGET_NAME,
session_id=42,
silent = True
)
| Description | Value | |
|---|---|---|
| 0 | session_id | 42 |
| 1 | Target | target |
| 2 | Original Data | (500, 311) |
| 3 | Missing Values | False |
| 4 | Numeric Features | 310 |
| 5 | Categorical Features | 0 |
| 6 | Ordinal Features | False |
| 7 | High Cardinality Features | False |
| 8 | High Cardinality Method | None |
| 9 | Transformed Train Set | (349, 310) |
| 10 | Transformed Test Set | (151, 310) |
| 11 | Shuffle Train-Test | True |
| 12 | Stratify Train-Test | False |
| 13 | Fold Generator | KFold |
| 14 | Fold Number | 10 |
| 15 | CPU Jobs | -1 |
| 16 | Use GPU | False |
| 17 | Log Experiment | False |
| 18 | Experiment Name | reg-default-name |
| 19 | USI | 4790 |
| 20 | Imputation Type | simple |
| 21 | Iterative Imputation Iteration | None |
| 22 | Numeric Imputer | mean |
| 23 | Iterative Imputation Numeric Model | None |
| 24 | Categorical Imputer | constant |
| 25 | Iterative Imputation Categorical Model | None |
| 26 | Unknown Categoricals Handling | least_frequent |
| 27 | Normalize | False |
| 28 | Normalize Method | None |
| 29 | Transformation | False |
| 30 | Transformation Method | None |
| 31 | PCA | False |
| 32 | PCA Method | None |
| 33 | PCA Components | None |
| 34 | Ignore Low Variance | False |
| 35 | Combine Rare Levels | False |
| 36 | Rare Level Threshold | None |
| 37 | Numeric Binning | False |
| 38 | Remove Outliers | False |
| 39 | Outliers Threshold | None |
| 40 | Remove Multicollinearity | False |
| 41 | Multicollinearity Threshold | None |
| 42 | Clustering | False |
| 43 | Clustering Iteration | None |
| 44 | Polynomial Features | False |
| 45 | Polynomial Degree | None |
| 46 | Trignometry Features | False |
| 47 | Polynomial Threshold | None |
| 48 | Group Features | False |
| 49 | Feature Selection | False |
| 50 | Feature Selection Method | classic |
| 51 | Features Selection Threshold | None |
| 52 | Feature Interaction | False |
| 53 | Feature Ratio | False |
| 54 | Interaction Threshold | None |
| 55 | Transform Target | False |
| 56 | Transform Target Method | box-cox |
# Submissions are scored by spearman correlation
def spearman(y_true, y_pred):
ret_score = spearmanr(y_true, y_pred)[0]
return ret_score if not np.isnan(ret_score) else 0.0
def pearson(y_true, y_pred):
ret_score = pearsonr(y_true, y_pred)[0]
return ret_score if not np.isnan(ret_score) else 0.0
spearman_corr = make_scorer(spearman, needs_proba=False)
pearson_corr = make_scorer(pearson, needs_proba=False)
add_metric('pearson', 'CORR', pearson)
add_metric('spear', 'SPEAR', spearman)
Name SPEAR
Display Name SPEAR
Score Function <function spearman at 0x0000019EFE4F0310>
Scorer make_scorer(spearman)
Target pred
Args {}
Greater is Better True
Custom True
Name: spear, dtype: object
get_metrics()
| Name | Display Name | Score Function | Scorer | Target | Args | Greater is Better | Custom | |
|---|---|---|---|---|---|---|---|---|
| ID | ||||||||
| mae | MAE | MAE | <function mean_absolute_error at 0x0000019EF9A... | neg_mean_absolute_error | pred | {} | False | False |
| mse | MSE | MSE | <function mean_squared_error at 0x0000019EF9A0... | neg_mean_squared_error | pred | {} | False | False |
| rmse | RMSE | RMSE | <function mean_squared_error at 0x0000019EF9A0... | neg_root_mean_squared_error | pred | {'squared': False} | False | False |
| r2 | R2 | R2 | <function r2_score at 0x0000019EF9A05A60> | r2 | pred | {} | True | False |
| rmsle | RMSLE | RMSLE | <function RMSLEMetricContainer.__init__.<local... | make_scorer(root_mean_squared_log_error, great... | pred | {} | False | False |
| mape | MAPE | MAPE | <function MAPEMetricContainer.__init__.<locals... | make_scorer(mean_absolute_percentage_error, gr... | pred | {} | False | False |
| pearson | CORR | CORR | <function pearson at 0x0000019EFE4F0550> | make_scorer(pearson) | pred | {} | True | True |
| spear | SPEAR | SPEAR | <function spearman at 0x0000019EFE4F0310> | make_scorer(spearman) | pred | {} | True | True |
xgb = create_model('xgboost', fold = 5)
| MAE | MSE | RMSE | R2 | RMSLE | MAPE | CORR | SPEAR | |
|---|---|---|---|---|---|---|---|---|
| 0 | 0.1677 | 0.0580 | 0.2409 | -0.3001 | 0.1686 | 0.3182 | 0.0235 | 0.1182 |
| 1 | 0.2180 | 0.0798 | 0.2825 | -0.7472 | 0.1818 | 0.3694 | -0.1615 | -0.1551 |
| 2 | 0.2124 | 0.0680 | 0.2607 | -0.1435 | 0.1697 | 0.4688 | 0.0718 | 0.0193 |
| 3 | 0.1803 | 0.0590 | 0.2429 | -0.1409 | 0.1754 | 0.3664 | 0.2891 | 0.2976 |
| 4 | 0.1930 | 0.0593 | 0.2435 | -0.0715 | 0.1689 | 0.3733 | 0.1697 | 0.1444 |
| Mean | 0.1943 | 0.0648 | 0.2541 | -0.2807 | 0.1729 | 0.3792 | 0.0785 | 0.0849 |
| SD | 0.0190 | 0.0083 | 0.0159 | 0.2450 | 0.0051 | 0.0491 | 0.1506 | 0.1495 |
tuned_xgb = tune_model(xgb, optimize='spear', fold= 5, n_iter = 10, search_library="optuna")
| MAE | MSE | RMSE | R2 | RMSLE | MAPE | CORR | SPEAR | |
|---|---|---|---|---|---|---|---|---|
| 0 | 0.1286 | 0.0446 | 0.2113 | -0.0000 | 0.1475 | 0.2349 | -0.0111 | 0.0088 |
| 1 | 0.1714 | 0.0571 | 0.2390 | -0.2511 | 0.1498 | 0.2730 | 0.0715 | 0.0511 |
| 2 | 0.1714 | 0.0607 | 0.2464 | -0.0215 | 0.1575 | 0.3648 | 0.0867 | 0.0992 |
| 3 | 0.1500 | 0.0536 | 0.2315 | -0.0356 | 0.1668 | 0.3047 | 0.0144 | 0.0618 |
| 4 | 0.1667 | 0.0562 | 0.2370 | -0.0152 | 0.1618 | 0.2975 | 0.1831 | 0.1981 |
| Mean | 0.1576 | 0.0544 | 0.2330 | -0.0647 | 0.1567 | 0.2950 | 0.0689 | 0.0838 |
| SD | 0.0165 | 0.0054 | 0.0119 | 0.0939 | 0.0072 | 0.0426 | 0.0674 | 0.0640 |
tournament_data[PREDICTION_NAME] = tuned_xgb.predict(tournament_data[feature_names[:-1]])
tournament_data[PREDICTION_NAME].to_csv("submission.csv", header=True)
Note: For a more in-depth treatment of Numerai in general, please refer to their sample notebook