About

I put together this notebook on how to use custom metrics with PyCaret, specifically for the Numerai compeittion.

import numpy as np
import pandas as pd
import csv
from pathlib import Path
from sklearn.metrics import make_scorer
from scipy.stats import spearmanr, pearsonr
from pycaret.regression import *

TARGET_NAME = f"target"
PREDICTION_NAME = f"prediction"

training_data = read_csv("numerai_training_data.csv")
tournament_data = read_csv("numerai_tournament_data.csv")

# Sampling for ease of demo/repeatability - obviously would use full set for competition
training_data = training_data.sample(500)
tournament_data = tournament_data.sample(500)

feature_names = [f for f in training_data.columns if f.startswith("feature") or f  == TARGET_NAME]
training_data.head(5)
era data_type feature_intelligence1 feature_intelligence2 feature_intelligence3 feature_intelligence4 feature_intelligence5 feature_intelligence6 feature_intelligence7 feature_intelligence8 feature_intelligence9 feature_intelligence10 feature_intelligence11 feature_intelligence12 feature_charisma1 feature_charisma2 feature_charisma3 feature_charisma4 feature_charisma5 feature_charisma6 feature_charisma7 feature_charisma8 feature_charisma9 feature_charisma10 feature_charisma11 feature_charisma12 feature_charisma13 feature_charisma14 feature_charisma15 feature_charisma16 feature_charisma17 feature_charisma18 feature_charisma19 feature_charisma20 feature_charisma21 feature_charisma22 feature_charisma23 feature_charisma24 feature_charisma25 feature_charisma26 feature_charisma27 feature_charisma28 feature_charisma29 feature_charisma30 feature_charisma31 feature_charisma32 feature_charisma33 feature_charisma34 feature_charisma35 feature_charisma36 feature_charisma37 feature_charisma38 feature_charisma39 feature_charisma40 feature_charisma41 feature_charisma42 feature_charisma43 feature_charisma44 feature_charisma45 feature_charisma46 feature_charisma47 feature_charisma48 feature_charisma49 feature_charisma50 feature_charisma51 feature_charisma52 feature_charisma53 feature_charisma54 feature_charisma55 feature_charisma56 feature_charisma57 feature_charisma58 feature_charisma59 feature_charisma60 feature_charisma61 feature_charisma62 feature_charisma63 feature_charisma64 feature_charisma65 feature_charisma66 feature_charisma67 feature_charisma68 feature_charisma69 feature_charisma70 feature_charisma71 feature_charisma72 feature_charisma73 feature_charisma74 feature_charisma75 feature_charisma76 feature_charisma77 feature_charisma78 feature_charisma79 feature_charisma80 feature_charisma81 feature_charisma82 feature_charisma83 feature_charisma84 feature_charisma85 feature_charisma86 feature_strength1 feature_strength2 feature_strength3 feature_strength4 feature_strength5 feature_strength6 feature_strength7 feature_strength8 feature_strength9 feature_strength10 feature_strength11 feature_strength12 feature_strength13 feature_strength14 feature_strength15 feature_strength16 feature_strength17 feature_strength18 feature_strength19 feature_strength20 feature_strength21 feature_strength22 feature_strength23 feature_strength24 feature_strength25 feature_strength26 feature_strength27 feature_strength28 feature_strength29 feature_strength30 feature_strength31 feature_strength32 feature_strength33 feature_strength34 feature_strength35 feature_strength36 feature_strength37 feature_strength38 feature_dexterity1 feature_dexterity2 feature_dexterity3 feature_dexterity4 feature_dexterity5 feature_dexterity6 feature_dexterity7 feature_dexterity8 feature_dexterity9 feature_dexterity10 feature_dexterity11 feature_dexterity12 feature_dexterity13 feature_dexterity14 feature_constitution1 feature_constitution2 feature_constitution3 feature_constitution4 feature_constitution5 feature_constitution6 feature_constitution7 feature_constitution8 feature_constitution9 feature_constitution10 feature_constitution11 feature_constitution12 feature_constitution13 feature_constitution14 feature_constitution15 feature_constitution16 feature_constitution17 feature_constitution18 feature_constitution19 feature_constitution20 feature_constitution21 feature_constitution22 feature_constitution23 feature_constitution24 feature_constitution25 feature_constitution26 feature_constitution27 feature_constitution28 feature_constitution29 feature_constitution30 feature_constitution31 feature_constitution32 feature_constitution33 feature_constitution34 feature_constitution35 feature_constitution36 feature_constitution37 feature_constitution38 feature_constitution39 feature_constitution40 feature_constitution41 feature_constitution42 feature_constitution43 feature_constitution44 feature_constitution45 feature_constitution46 feature_constitution47 feature_constitution48 feature_constitution49 feature_constitution50 feature_constitution51 feature_constitution52 feature_constitution53 feature_constitution54 feature_constitution55 feature_constitution56 feature_constitution57 feature_constitution58 feature_constitution59 feature_constitution60 feature_constitution61 feature_constitution62 feature_constitution63 feature_constitution64 feature_constitution65 feature_constitution66 feature_constitution67 feature_constitution68 feature_constitution69 feature_constitution70 feature_constitution71 feature_constitution72 feature_constitution73 feature_constitution74 feature_constitution75 feature_constitution76 feature_constitution77 feature_constitution78 feature_constitution79 feature_constitution80 feature_constitution81 feature_constitution82 feature_constitution83 feature_constitution84 feature_constitution85 feature_constitution86 feature_constitution87 feature_constitution88 feature_constitution89 feature_constitution90 feature_constitution91 feature_constitution92 feature_constitution93 feature_constitution94 feature_constitution95 feature_constitution96 feature_constitution97 feature_constitution98 feature_constitution99 feature_constitution100 feature_constitution101 feature_constitution102 feature_constitution103 feature_constitution104 feature_constitution105 feature_constitution106 feature_constitution107 feature_constitution108 feature_constitution109 feature_constitution110 feature_constitution111 feature_constitution112 feature_constitution113 feature_constitution114 feature_wisdom1 feature_wisdom2 feature_wisdom3 feature_wisdom4 feature_wisdom5 feature_wisdom6 feature_wisdom7 feature_wisdom8 feature_wisdom9 feature_wisdom10 feature_wisdom11 feature_wisdom12 feature_wisdom13 feature_wisdom14 feature_wisdom15 feature_wisdom16 feature_wisdom17 feature_wisdom18 feature_wisdom19 feature_wisdom20 feature_wisdom21 feature_wisdom22 feature_wisdom23 feature_wisdom24 feature_wisdom25 feature_wisdom26 feature_wisdom27 feature_wisdom28 feature_wisdom29 feature_wisdom30 feature_wisdom31 feature_wisdom32 feature_wisdom33 feature_wisdom34 feature_wisdom35 feature_wisdom36 feature_wisdom37 feature_wisdom38 feature_wisdom39 feature_wisdom40 feature_wisdom41 feature_wisdom42 feature_wisdom43 feature_wisdom44 feature_wisdom45 feature_wisdom46 target
id
n6346c8f8784a565 era75 train 0.75 1.00 1.00 0.50 0.50 0.50 0.25 1.00 0.75 0.75 0.75 0.50 1.00 0.25 0.50 0.00 0.00 0.75 0.25 1.00 0.25 0.25 1.00 0.50 0.75 0.00 1.00 0.25 0.00 1.00 1.00 0.00 0.75 0.00 0.75 0.25 1.00 0.00 0.00 0.75 0.00 0.00 0.5 1.00 0.00 0.25 0.00 0.75 1.00 0.00 0.50 0.00 0.00 0.75 1.00 0.00 0.50 0.50 1.0 1.00 0.25 0.50 0.50 1.00 0.75 0.75 0.75 0.0 0.00 1.00 0.00 0.00 0.0 0.75 0.75 0.00 0.00 0.00 0.0 0.50 0.25 0.75 0.0 0.25 0.25 0.00 0.75 1.00 0.00 0.25 0.00 0.75 1.00 0.25 0.00 1.00 0.5 0.75 0.00 0.00 1.0 0.00 0.75 0.5 0.00 0.75 0.00 1.00 0.00 0.75 0.25 1.00 1.00 0.00 0.50 1.0 1.0 0.75 0.00 0.00 0.00 1.00 0.00 0.00 1.00 0.75 1.00 0.75 0.00 0.00 0.00 1.00 0.00 1.00 1.0 0.75 0.25 0.75 1.0 1.00 0.75 1.00 1.00 1.00 0.5 1.0 0.75 1.00 1.0 1.00 0.25 0.25 1.00 0.00 0.5 0.5 1.00 0.00 1.00 0.5 0.0 0.75 1.0 0.75 0.0 0.00 0.00 0.00 0.00 0.75 0.50 0.0 1.0 0.50 0.25 0.75 1.0 0.00 1.00 0.50 0.75 0.25 1.00 0.0 0.00 0.00 0.00 0.25 0.0 0.00 0.00 0.75 0.75 1.0 0.00 0.25 0.00 0.00 0.00 1.00 0.50 1.00 0.75 0.0 1.0 0.00 0.25 0.00 0.50 0.5 0.75 0.50 0.0 0.00 0.00 1.0 0.50 0.0 1.00 0.75 0.75 0.25 0.25 0.50 0.00 0.25 0.0 0.00 1.0 0.50 0.75 0.50 0.00 0.75 0.75 0.75 0.5 0.00 0.00 0.0 0.00 0.25 1.0 1.00 0.00 1.00 0.00 0.25 0.00 0.00 0.00 0.00 0.25 0.5 0.00 0.0 0.00 0.25 0.0 0.0 0.25 0.25 0.00 0.5 0.00 0.00 0.00 0.25 0.25 0.0 0.25 0.00 0.00 0.0 0.25 0.00 0.00 0.00 0.00 0.25 0.00 0.00 0.00 0.00 0.00 0.25 0.00 0.00 0.25 0.00 0.00 0.25 0.00 0.25 0.00 0.25 0.00 0.25 0.00 0.50 0.00 0.00 0.00 0.50 0.50 0.75 0.00 0.25 0.00 0.25 0.25
n46dccf4bc8006a8 era75 train 0.00 1.00 1.00 0.00 0.25 0.50 0.25 0.00 1.00 1.00 0.00 0.25 0.25 0.75 0.25 1.00 0.50 1.00 1.00 0.75 0.00 0.25 0.50 1.00 0.75 1.00 0.25 1.00 1.00 1.00 0.50 0.25 0.75 0.25 0.75 0.75 0.25 1.00 0.75 0.75 0.75 0.75 1.0 1.00 1.00 1.00 1.00 1.00 0.75 1.00 0.75 0.75 0.75 1.00 1.00 1.00 0.25 0.50 0.0 0.75 0.50 1.00 1.00 0.75 0.75 1.00 1.00 0.5 0.50 0.75 0.50 0.25 0.5 1.00 1.00 0.75 0.50 0.75 1.0 0.75 0.25 1.00 1.0 1.00 0.25 1.00 1.00 1.00 0.75 0.50 0.75 1.00 1.00 0.75 0.50 0.75 1.0 0.75 0.50 0.50 0.0 0.75 0.50 0.5 0.50 0.25 0.50 0.50 0.50 0.25 0.75 0.75 0.75 1.00 0.75 0.5 0.0 0.50 0.75 0.50 0.50 0.00 0.00 0.50 0.75 0.75 0.00 0.50 0.25 0.25 0.50 0.75 0.75 0.75 0.5 0.50 1.00 1.00 1.0 0.75 1.00 1.00 1.00 1.00 1.0 1.0 1.00 0.00 1.0 0.25 0.25 1.00 0.25 0.75 0.0 0.0 0.00 0.25 0.75 0.0 1.0 0.00 0.0 0.75 0.5 0.50 0.25 0.25 1.00 0.75 0.25 0.5 0.0 0.75 0.50 0.75 0.0 0.75 0.25 0.75 1.00 0.50 1.00 0.5 0.75 0.75 0.50 0.25 0.5 0.50 0.00 0.25 0.50 0.0 1.00 0.00 1.00 0.25 1.00 0.50 1.00 1.00 0.50 1.0 0.0 0.25 0.25 1.00 1.00 1.0 1.00 0.50 0.5 0.75 0.00 0.0 0.75 1.0 1.00 0.50 0.75 0.00 1.00 0.25 0.25 0.75 0.5 0.50 0.0 0.50 0.75 0.50 0.50 0.75 0.75 0.75 1.0 0.25 0.50 0.0 0.00 0.50 0.0 0.75 1.00 1.00 0.25 0.75 0.75 0.75 0.25 0.25 0.25 1.0 0.50 0.5 0.25 0.00 0.5 0.0 0.50 0.00 0.75 0.5 0.00 0.50 0.50 0.25 0.25 0.0 0.50 0.25 0.50 0.5 0.25 0.50 0.75 0.50 0.25 0.25 0.25 0.75 0.25 0.25 0.25 0.50 0.75 0.00 0.50 0.00 0.25 0.25 0.25 0.25 0.25 0.50 0.00 0.75 0.75 0.00 0.25 0.00 0.25 0.75 0.75 0.00 0.25 0.25 0.50 0.50 0.25
n3e1605f37041c0c era6 train 0.75 0.25 0.00 0.75 0.75 0.50 0.25 0.75 0.00 0.00 0.75 1.00 0.00 0.50 0.75 0.00 0.75 1.00 0.50 0.50 1.00 1.00 0.25 0.75 0.00 0.75 0.50 0.25 0.00 0.00 0.00 0.75 0.00 0.25 1.00 0.75 0.75 0.25 0.25 0.00 1.00 0.25 0.0 0.00 0.75 0.50 0.50 0.00 0.00 0.25 1.00 1.00 1.00 1.00 0.00 0.25 0.75 0.25 1.0 0.00 0.75 0.00 0.00 0.00 0.00 1.00 0.50 0.5 1.00 0.25 1.00 0.25 1.0 0.25 0.00 0.75 0.75 0.00 0.0 0.00 1.00 0.00 1.0 0.50 0.75 0.25 1.00 0.00 0.00 1.00 0.75 0.50 0.00 0.00 0.75 1.00 0.0 1.00 1.00 0.00 1.0 1.00 0.50 0.5 0.50 0.50 0.50 0.75 1.00 0.25 0.50 1.00 0.75 0.25 0.50 1.0 1.0 0.50 1.00 0.75 0.00 1.00 0.00 0.50 0.50 0.75 0.75 0.50 0.00 0.00 0.50 0.75 0.50 0.75 1.0 0.50 0.00 0.00 0.0 0.25 0.00 0.00 0.25 0.00 0.0 0.0 0.00 0.50 0.0 0.50 0.75 0.00 0.00 0.00 1.0 1.0 0.50 0.25 0.00 1.0 0.0 1.00 0.0 0.00 0.0 0.50 0.25 0.75 0.00 0.25 0.25 0.5 0.0 0.00 0.25 0.75 0.5 0.00 1.00 0.00 0.25 0.50 0.00 0.0 0.00 0.25 0.00 0.50 0.5 0.50 0.50 0.00 0.25 0.5 0.00 1.00 0.25 0.75 0.25 1.00 0.00 1.00 0.50 0.0 0.5 1.00 0.75 1.00 0.00 0.0 0.00 0.25 0.0 0.25 0.00 0.5 0.25 1.0 0.75 0.75 0.75 0.00 0.00 1.00 0.75 0.50 0.5 0.00 0.5 1.00 0.00 0.00 0.50 0.75 0.75 1.00 0.0 0.00 0.50 0.0 1.00 0.25 0.5 1.00 0.00 0.00 0.25 0.00 0.25 0.50 1.00 1.00 0.00 0.0 0.00 0.5 0.50 0.50 0.0 1.0 0.25 1.00 0.25 0.5 0.00 0.25 1.00 0.00 0.00 1.0 0.00 0.00 1.00 0.0 1.00 0.25 0.00 0.00 0.00 0.00 0.25 0.00 0.00 0.00 0.50 1.00 0.00 1.00 0.00 0.75 0.00 0.50 0.00 0.25 0.00 0.00 0.75 0.00 0.00 0.25 0.25 0.25 0.00 0.00 0.00 0.25 0.75 0.00 0.00 0.00 0.25
na662ad0039c3091 era107 train 0.50 0.75 0.75 0.25 0.50 0.75 0.75 0.00 0.75 0.75 0.25 0.50 0.25 0.25 1.00 0.75 0.25 1.00 1.00 0.75 0.50 1.00 0.75 0.50 0.50 0.50 0.75 0.50 0.75 0.25 0.75 0.25 1.00 0.50 0.50 0.25 0.25 0.25 0.50 0.25 0.50 0.25 0.5 0.25 0.50 1.00 0.50 1.00 0.25 0.25 0.50 0.50 0.25 0.50 1.00 0.50 1.00 1.00 0.0 0.00 1.00 0.25 0.25 0.50 0.25 0.75 1.00 1.0 0.25 0.75 0.25 0.75 0.5 0.75 1.00 0.25 1.00 1.00 0.5 0.50 0.75 1.00 0.5 1.00 0.75 0.50 0.50 0.25 0.25 1.00 0.25 0.25 0.25 0.75 0.00 0.25 1.0 0.25 0.75 0.50 0.5 0.75 1.00 0.5 0.25 0.50 0.25 0.75 0.25 0.50 1.00 1.00 0.50 0.25 0.50 1.0 0.5 0.50 0.25 0.25 0.25 0.75 0.25 0.75 0.50 0.25 0.75 0.50 0.00 0.25 0.75 1.00 0.25 0.50 1.0 0.50 0.75 0.75 0.5 0.00 0.50 0.25 0.00 0.50 0.5 0.5 0.50 0.00 0.5 0.25 0.50 0.75 0.00 0.00 0.0 0.0 0.75 0.25 0.75 0.0 0.5 0.50 0.5 0.25 0.5 0.25 0.00 0.00 0.50 1.00 0.00 0.0 0.0 0.75 0.00 1.00 1.0 0.00 0.50 0.25 0.50 0.25 0.75 0.0 0.25 0.25 0.75 0.75 0.0 0.25 0.25 0.25 1.00 1.0 0.75 0.00 0.50 0.00 0.25 0.25 0.25 0.75 0.75 1.0 1.0 0.25 0.25 1.00 0.25 1.0 1.00 1.00 0.5 0.25 0.25 1.0 0.25 1.0 1.00 0.00 1.00 0.00 0.50 0.00 0.25 0.25 0.0 0.25 0.5 0.75 0.25 0.75 0.25 1.00 1.00 1.00 1.0 0.00 0.00 0.0 0.25 1.00 1.0 1.00 1.00 0.75 0.00 0.25 1.00 0.25 0.50 0.25 0.25 0.5 0.25 0.5 0.00 0.25 0.0 0.5 0.25 0.00 0.00 1.0 0.75 0.75 0.75 1.00 1.00 0.5 1.00 0.00 1.00 0.0 0.25 0.75 1.00 0.75 0.25 0.75 1.00 0.75 0.00 1.00 0.50 1.00 0.50 0.75 0.50 0.25 0.50 0.00 0.75 1.00 0.50 0.25 1.00 1.00 1.00 0.75 0.75 0.75 0.50 0.75 1.00 0.25 1.00 0.50 0.75 1.00 0.50
nf324eff3bcbf017 era99 train 0.25 0.25 0.50 0.75 0.00 0.25 0.50 0.75 0.50 0.50 0.75 0.25 0.50 0.50 0.00 0.00 0.75 1.00 0.25 0.00 1.00 0.75 0.00 0.25 0.25 0.00 0.00 0.50 0.00 0.00 0.50 0.75 0.00 0.00 0.00 0.50 0.50 0.75 0.50 0.00 1.00 0.25 0.0 0.00 0.00 1.00 0.75 0.00 0.00 0.50 0.00 1.00 0.25 1.00 0.25 1.00 0.25 0.00 0.5 0.00 0.00 0.00 0.00 0.00 0.00 1.00 1.00 0.0 0.75 0.25 0.25 0.00 1.0 0.25 0.00 0.75 0.00 0.25 0.5 0.00 1.00 0.25 1.0 0.50 0.00 1.00 1.00 0.00 0.50 1.00 0.75 0.25 0.00 0.00 0.75 1.00 0.0 1.00 1.00 0.25 1.0 1.00 0.00 0.5 0.50 0.75 0.25 1.00 0.50 1.00 0.00 0.00 1.00 0.50 0.50 1.0 1.0 0.50 0.50 0.00 0.25 1.00 0.00 0.50 0.50 0.75 1.00 1.00 0.00 0.25 0.50 0.00 0.50 1.00 1.0 0.50 0.50 0.50 0.0 0.00 0.25 0.00 0.00 0.25 0.5 0.5 0.00 0.25 0.0 0.00 0.75 0.00 0.50 0.00 0.0 0.5 1.00 0.00 0.00 0.5 0.0 1.00 0.5 0.00 1.0 0.00 0.00 1.00 0.25 0.75 0.00 0.5 0.5 0.00 0.50 0.75 1.0 0.00 1.00 0.00 0.25 1.00 0.00 0.0 0.25 0.25 1.00 1.00 1.0 0.00 0.50 0.00 0.75 0.5 0.25 1.00 0.25 0.50 0.25 1.00 0.00 0.75 1.00 0.0 0.5 1.00 1.00 0.25 0.00 0.0 0.00 0.75 0.0 0.50 0.00 0.5 0.50 0.0 1.00 0.00 1.00 0.50 0.00 0.00 1.00 0.25 0.5 0.00 0.5 0.75 0.00 0.00 0.25 0.75 0.75 0.50 0.0 0.50 0.25 0.5 1.00 1.00 0.5 0.50 0.25 0.00 0.50 0.50 1.00 0.25 0.75 1.00 0.50 0.0 0.00 1.0 0.50 0.50 0.0 1.0 0.50 1.00 0.50 1.0 0.00 0.00 0.50 0.75 0.00 0.0 0.00 0.50 0.75 0.5 0.25 0.00 0.25 0.00 0.00 1.00 0.25 0.00 0.50 0.00 0.25 0.75 0.00 0.50 0.00 1.00 0.00 0.00 0.25 1.00 0.00 0.00 0.50 0.00 0.00 0.50 0.25 0.00 0.00 0.00 0.00 0.25 0.50 0.00 1.00 0.00 0.25
reg = setup(
    data=training_data[feature_names],
    target=TARGET_NAME,
    session_id=42,
    silent = True
)

Description Value
0 session_id 42
1 Target target
2 Original Data (500, 311)
3 Missing Values False
4 Numeric Features 310
5 Categorical Features 0
6 Ordinal Features False
7 High Cardinality Features False
8 High Cardinality Method None
9 Transformed Train Set (349, 310)
10 Transformed Test Set (151, 310)
11 Shuffle Train-Test True
12 Stratify Train-Test False
13 Fold Generator KFold
14 Fold Number 10
15 CPU Jobs -1
16 Use GPU False
17 Log Experiment False
18 Experiment Name reg-default-name
19 USI 4790
20 Imputation Type simple
21 Iterative Imputation Iteration None
22 Numeric Imputer mean
23 Iterative Imputation Numeric Model None
24 Categorical Imputer constant
25 Iterative Imputation Categorical Model None
26 Unknown Categoricals Handling least_frequent
27 Normalize False
28 Normalize Method None
29 Transformation False
30 Transformation Method None
31 PCA False
32 PCA Method None
33 PCA Components None
34 Ignore Low Variance False
35 Combine Rare Levels False
36 Rare Level Threshold None
37 Numeric Binning False
38 Remove Outliers False
39 Outliers Threshold None
40 Remove Multicollinearity False
41 Multicollinearity Threshold None
42 Clustering False
43 Clustering Iteration None
44 Polynomial Features False
45 Polynomial Degree None
46 Trignometry Features False
47 Polynomial Threshold None
48 Group Features False
49 Feature Selection False
50 Feature Selection Method classic
51 Features Selection Threshold None
52 Feature Interaction False
53 Feature Ratio False
54 Interaction Threshold None
55 Transform Target False
56 Transform Target Method box-cox

Adding Metrics

PyCaret makes use of scikit-learn's make_scorer. After initialization of the PyCaret regressor/classifier module, you simply utilize add_metric to attach them to PyCaret's list of metrics.

# Submissions are scored by spearman correlation

def spearman(y_true, y_pred):
    ret_score = spearmanr(y_true, y_pred)[0]
    return ret_score if not np.isnan(ret_score) else 0.0


def pearson(y_true, y_pred):
    ret_score = pearsonr(y_true, y_pred)[0]
    return ret_score if not np.isnan(ret_score) else 0.0

spearman_corr = make_scorer(spearman, needs_proba=False)
pearson_corr = make_scorer(pearson, needs_proba=False)

add_metric('pearson', 'CORR', pearson)
add_metric('spear', 'SPEAR', spearman)
Name                                                     SPEAR
Display Name                                             SPEAR
Score Function       <function spearman at 0x0000019EFE4F0310>
Scorer                                   make_scorer(spearman)
Target                                                    pred
Args                                                        {}
Greater is Better                                         True
Custom                                                    True
Name: spear, dtype: object
get_metrics()
Name Display Name Score Function Scorer Target Args Greater is Better Custom
ID
mae MAE MAE <function mean_absolute_error at 0x0000019EF9A... neg_mean_absolute_error pred {} False False
mse MSE MSE <function mean_squared_error at 0x0000019EF9A0... neg_mean_squared_error pred {} False False
rmse RMSE RMSE <function mean_squared_error at 0x0000019EF9A0... neg_root_mean_squared_error pred {'squared': False} False False
r2 R2 R2 <function r2_score at 0x0000019EF9A05A60> r2 pred {} True False
rmsle RMSLE RMSLE <function RMSLEMetricContainer.__init__.<local... make_scorer(root_mean_squared_log_error, great... pred {} False False
mape MAPE MAPE <function MAPEMetricContainer.__init__.<locals... make_scorer(mean_absolute_percentage_error, gr... pred {} False False
pearson CORR CORR <function pearson at 0x0000019EFE4F0550> make_scorer(pearson) pred {} True True
spear SPEAR SPEAR <function spearman at 0x0000019EFE4F0310> make_scorer(spearman) pred {} True True

Running the Models with New Metrics

Adding these metrics allow them to be shown in all new model creation by PyCaret. Simply optimize when using tune_model and it will optimize by your new metric.

xgb = create_model('xgboost', fold = 5)
MAE MSE RMSE R2 RMSLE MAPE CORR SPEAR
0 0.1677 0.0580 0.2409 -0.3001 0.1686 0.3182 0.0235 0.1182
1 0.2180 0.0798 0.2825 -0.7472 0.1818 0.3694 -0.1615 -0.1551
2 0.2124 0.0680 0.2607 -0.1435 0.1697 0.4688 0.0718 0.0193
3 0.1803 0.0590 0.2429 -0.1409 0.1754 0.3664 0.2891 0.2976
4 0.1930 0.0593 0.2435 -0.0715 0.1689 0.3733 0.1697 0.1444
Mean 0.1943 0.0648 0.2541 -0.2807 0.1729 0.3792 0.0785 0.0849
SD 0.0190 0.0083 0.0159 0.2450 0.0051 0.0491 0.1506 0.1495
tuned_xgb = tune_model(xgb, optimize='spear', fold= 5, n_iter = 10, search_library="optuna")
MAE MSE RMSE R2 RMSLE MAPE CORR SPEAR
0 0.1286 0.0446 0.2113 -0.0000 0.1475 0.2349 -0.0111 0.0088
1 0.1714 0.0571 0.2390 -0.2511 0.1498 0.2730 0.0715 0.0511
2 0.1714 0.0607 0.2464 -0.0215 0.1575 0.3648 0.0867 0.0992
3 0.1500 0.0536 0.2315 -0.0356 0.1668 0.3047 0.0144 0.0618
4 0.1667 0.0562 0.2370 -0.0152 0.1618 0.2975 0.1831 0.1981
Mean 0.1576 0.0544 0.2330 -0.0647 0.1567 0.2950 0.0689 0.0838
SD 0.0165 0.0054 0.0119 0.0939 0.0072 0.0426 0.0674 0.0640
tournament_data[PREDICTION_NAME] = tuned_xgb.predict(tournament_data[feature_names[:-1]])
tournament_data[PREDICTION_NAME].to_csv("submission.csv", header=True)

Note: For a more in-depth treatment of Numerai in general, please refer to their sample notebook