Source code for h2o.model.metrics_base

# -*- encoding: utf-8 -*-
"""

:copyright: (c) 2016 H2O.ai
:license:   Apache License Version 2.0 (see LICENSE for details)
"""
import math
from collections import OrderedDict

from h2o.display import H2ODisplay, display, repr_def, format_to_html, format_to_multiline
from h2o.utils.compatibility import *  # NOQA
from h2o.utils.metaclass import backwards_compatibility, deprecated_fn, h2o_meta
from h2o.utils.typechecks import is_type, numeric


[docs]@backwards_compatibility( instance_attrs=dict( giniCoef=lambda self, *args, **kwargs: self.gini(*args, **kwargs) ) ) class MetricsBase(h2o_meta(H2ODisplay)): """ A parent class to house common metrics available for the various Metrics types. The methods here are available across different model categories. .. note:: This class and its subclasses are used at runtime as mixins: their methods can (and should) be accessed directly from a metrics object, for example as a result of :func:`~h2o.model.ModelBase.model_performance`. """ _on_mapping = OrderedDict( training_metrics='train', validation_metrics='validation', cross_validation_metrics='cross-validation', _='test' ) def __init__(self, metric_json, on=None, algo=""): self._metric_json = metric_json._metric_json if isinstance(metric_json, MetricsBase) else metric_json self._on = None self._algo = algo # assert on is None or on in MetricsBase._on_mapping self._on = MetricsBase._on_mapping.get(on or '_', None) if not self._on: raise ValueError("on param expected to be one of {accepted}, but got {on}: ".format( accepted=[k for k in MetricsBase._on_mapping if not k.startswith('_')], on=on ))
[docs] @classmethod def make(cls, kvs): """Factory method to instantiate a MetricsBase object from the list of key-value pairs.""" return cls(metric_json=dict(kvs))
# TODO: convert to actual fields list def __getitem__(self, key): return self._metric_json.get(key) @staticmethod def _has(dictionary, key): return key in dictionary and dictionary[key] is not None def _str_items(self, verbosity=None): # edge cases if self._metric_json is None: return "WARNING: Model metrics cannot be calculated, please check that the response column was correctly provided in your dataset." metric_type = self._metric_json['__meta']['schema_type'] # metric cond based on data distribution m_is_binomial = "Binomial" in metric_type m_is_multinomial = "Multinomial" in metric_type m_is_ordinal = "Ordinal" in metric_type m_is_regression = "Regression" in metric_type # metric cond based on algo m_is_anomaly = "Anomaly" in metric_type m_is_clustering = "Clustering" in metric_type m_is_generic = "Generic" in metric_type m_is_glm = "GLM" in metric_type m_is_hglm = "HGLM" in metric_type m_is_uplift = "Uplift" in metric_type # fixme: can't we rather check if each value is available instead of doing this weird logic? # we could have mixin extensions for algos like (H)GLM instead taking everything from this (not so) "base" class. # specific metric cond m_supports_logloss = (m_is_binomial or m_is_multinomial or m_is_ordinal) and not m_is_uplift m_supports_mpce = (m_is_binomial or m_is_multinomial or m_is_ordinal) and not (m_is_glm or m_is_uplift) # GLM excluded? m_supports_mse = not (m_is_anomaly or m_is_clustering or m_is_uplift) m_supports_r2 = m_is_regression and m_is_glm items = [ "{mtype}: {algo}".format(mtype=metric_type, algo=self._algo), "** Reported on {} data. **".format(self._on), "", ] if self.custom_metric_name(): # adding on top: if users specifies a custom metric, it needs to be highlighted. items.append("{name}: {value}".format(name=self.custom_metric_name(), value=self.custom_metric_value())) if m_supports_mse: items.extend([ "MSE: {}".format(self.mse()), "RMSE: {}".format(self.rmse()), ]) if m_is_regression: items.extend([ "MAE: {}".format(self.mae()), "RMSLE: {}".format(self.rmsle()), "Mean Residual Deviance: {}".format(self.mean_residual_deviance()), ]) if m_supports_r2: items.append("R^2: {}".format(self.r2())) if m_supports_logloss: items.append("LogLoss: {}".format(self.logloss())) if m_supports_mpce: items.append("Mean Per-Class Error: {}".format(self._mean_per_class_error())) if m_is_binomial and not m_is_uplift: # can be picked from H2OBinomialModelMetrics (refers to method not available in this class!) items.extend([ "AUC: {}".format(self.auc()), "AUCPR: {}".format(self.aucpr()), "Gini: {}".format(self.gini()), ]) if m_is_multinomial: auc, aucpr = self.auc(), self.aucpr() if is_type(auc, numeric): items.append("AUC: {}".format(auc)) if is_type(aucpr, numeric): items.append("AUCPR: {}".format(aucpr)) if m_is_glm: if m_is_hglm and not m_is_generic: items.extend([ "Standard error of fixed columns: {}".format(self.hglm_metric("sefe")), "Standard error of random columns: {}".format(self.hglm_metric("sere")), "Coefficients for fixed columns: {}".format(self.hglm_metric("fixedf")), "Coefficients for random columns: {}".format(self.hglm_metric("ranef")), "Random column indices: {}".format(self.hglm_metric("randc")), "Dispersion parameter of the mean model (residual variance for LMM): {}".format(self.hglm_metric("varfix")), "Dispersion parameter of the random columns (variance of random columns): {}".format(self.hglm_metric("varranef")), "Convergence reached for algorithm: {}".format(self.hglm_metric("converge")), "Deviance degrees of freedom for mean part of the model: {}".format(self.hglm_metric("dfrefe")), "Estimates and standard errors of the linear prediction in the dispersion model: {}".format(self.hglm_metric("summvc1")), "Estimates and standard errors of the linear predictor for the dispersion parameter of the random columns: {}".format(self.hglm_metric("summvc2")), "Index of most influential observation (-1 if none): {}".format(self.hglm_metric("bad")), "H-likelihood: {}".format(self.hglm_metric("hlik")), "Profile log-likelihood profiled over random columns: {}".format(self.hglm_metric("pvh")), "Adjusted profile log-likelihood profiled over fixed and random effects: {}".format(self.hglm_metric("pbvh")), "Conditional AIC: {}".format(self.hglm_metric("caic")), ]) else: items.extend([ "Null degrees of freedom: {}".format(self.null_degrees_of_freedom()), "Residual degrees of freedom: {}".format(self.residual_degrees_of_freedom()), "Null deviance: {}".format(self.null_deviance()), "Residual deviance: {}".format(self.residual_deviance()), ]) if m_is_glm: if is_type(self.aic(), numeric) and not math.isnan(self.aic()) and self.aic() != 0: items.append("AIC: {}".format(self.aic())) if is_type(self.loglikelihood(), numeric) and not math.isnan(self.loglikelihood()) and self.loglikelihood() != 0: items.append("Loglikelihood: {}".format(self.loglikelihood())) items.extend(self._str_items_custom()) return items def _str_items_custom(self): return [] def _repr_(self): return repr_def(self, attributes='all') def _str_(self, verbosity=None): items = self._str_items(verbosity) if isinstance(items, list): return format_to_multiline(items) return items def _str_html_(self, verbosity=None): items = self._str_items(verbosity) if isinstance(items, list): return format_to_html(items) return items
[docs] def show(self, verbosity=None, fmt=None): return display(self, fmt=fmt, verbosity=verbosity)
[docs] def r2(self): """The R squared coefficient. :examples: >>> from h2o.estimators.gbm import H2OGradientBoostingEstimator >>> cars = h2o.import_file("https://s3.amazonaws.com/h2o-public-test-data/smalldata/junit/cars_20mpg.csv") >>> cars["economy_20mpg"] = cars["economy_20mpg"].asfactor() >>> predictors = ["displacement","power","weight","acceleration","year"] >>> response = "economy_20mpg" >>> train, valid = cars.split_frame(ratios = [.8], seed = 1234) >>> cars_gbm = H2OGradientBoostingEstimator(seed = 1234) >>> cars_gbm.train(x = predictors, ... y = response, ... training_frame = train, ... validation_frame = valid) >>> cars_gbm.r2() """ return self._metric_json["r2"]
[docs] def logloss(self): """Log loss. :examples: >>> from h2o.estimators.gbm import H2OGradientBoostingEstimator >>> cars = h2o.import_file("https://s3.amazonaws.com/h2o-public-test-data/smalldata/junit/cars_20mpg.csv") >>> cars["economy_20mpg"] = cars["economy_20mpg"].asfactor() >>> predictors = ["displacement","power","weight","acceleration","year"] >>> response = "economy_20mpg" >>> train, valid = cars.split_frame(ratios = [.8], seed = 1234) >>> cars_gbm = H2OGradientBoostingEstimator(seed = 1234) >>> cars_gbm.train(x = predictors, ... y = response, ... training_frame = train, ... validation_frame = valid) >>> cars_gbm.logloss() """ return self._metric_json["logloss"]
[docs] def nobs(self): """ The number of observations. :examples: >>> from h2o.estimators.gbm import H2OGradientBoostingEstimator >>> cars = h2o.import_file("https://s3.amazonaws.com/h2o-public-test-data/smalldata/junit/cars_20mpg.csv") >>> cars["economy_20mpg"] = cars["economy_20mpg"].asfactor() >>> predictors = ["displacement","power","weight","acceleration","year"] >>> response = "economy_20mpg" >>> train, valid = cars.split_frame(ratios = [.8], seed = 1234) >>> cars_gbm = H2OGradientBoostingEstimator(seed = 1234) >>> cars_gbm.train(x = predictors, ... y = response, ... training_frame = train, ... validation_frame = valid) >>> perf = cars_gbm.model_performance() >>> perf.nobs() """ return self._metric_json["nobs"]
[docs] def mean_residual_deviance(self): """The mean residual deviance for this set of metrics. :examples: >>> from h2o.estimators.gbm import H2OGradientBoostingEstimator >>> airlines= h2o.import_file("https://s3.amazonaws.com/h2o-public-test-data/smalldata/airlines/AirlinesTest.csv.zip") >>> air_gbm = H2OGradientBoostingEstimator() >>> air_gbm.train(x=list(range(9)), ... y=9, ... training_frame=airlines, ... validation_frame=airlines) >>> air_gbm.mean_residual_deviance(train=True,valid=False,xval=False) """ return self._metric_json["mean_residual_deviance"]
[docs] def auc(self): """The AUC for this set of metrics. :examples: >>> from h2o.estimators.gbm import H2OGradientBoostingEstimator >>> cars = h2o.import_file("https://s3.amazonaws.com/h2o-public-test-data/smalldata/junit/cars_20mpg.csv") >>> cars["economy_20mpg"] = cars["economy_20mpg"].asfactor() >>> predictors = ["displacement","power","weight","acceleration","year"] >>> response = "economy_20mpg" >>> train, valid = cars.split_frame(ratios = [.8], seed = 1234) >>> cars_gbm = H2OGradientBoostingEstimator(seed = 1234) >>> cars_gbm.train(x = predictors, ... y = response, ... training_frame = train, ... validation_frame = valid) >>> cars_gbm.auc() """ return self._metric_json['AUC']
[docs] def aucpr(self): """The area under the precision recall curve. :examples: >>> from h2o.estimators.gbm import H2OGradientBoostingEstimator >>> cars = h2o.import_file("https://s3.amazonaws.com/h2o-public-test-data/smalldata/junit/cars_20mpg.csv") >>> cars["economy_20mpg"] = cars["economy_20mpg"].asfactor() >>> predictors = ["displacement","power","weight","acceleration","year"] >>> response = "economy_20mpg" >>> train, valid = cars.split_frame(ratios = [.8], seed = 1234) >>> cars_gbm = H2OGradientBoostingEstimator(seed = 1234) >>> cars_gbm.train(x = predictors, ... y = response, ... training_frame = train, ... validation_frame = valid) >>> cars_gbm.aucpr() """ return self._metric_json['pr_auc']
[docs] @deprecated_fn(replaced_by=aucpr) def pr_auc(self): pass
[docs] def aic(self): """The AIC for this set of metrics. :examples: >>> from h2o.estimators.glm import H2OGeneralizedLinearEstimator >>> prostate = h2o.import_file("http://s3.amazonaws.com/h2o-public-test-data/smalldata/prostate/prostate.csv.zip") >>> prostate[2] = prostate[2].asfactor() >>> prostate[4] = prostate[4].asfactor() >>> prostate[5] = prostate[5].asfactor() >>> prostate[8] = prostate[8].asfactor() >>> predictors = ["AGE","RACE","DPROS","DCAPS","PSA","VOL","GLEASON"] >>> response = "CAPSULE" >>> train, valid = prostate.split_frame(ratios=[.8],seed=1234) >>> pros_glm = H2OGeneralizedLinearEstimator(family="binomial") >>> pros_glm.train(x = predictors, ... y = response, ... training_frame = train, ... validation_frame = valid) >>> pros_glm.aic() """ return self._metric_json['AIC']
[docs] def loglikelihood(self): """The log likelihood for this set of metrics. :examples: >>> from h2o.estimators.glm import H2OGeneralizedLinearEstimator >>> prostate = h2o.import_file("http://s3.amazonaws.com/h2o-public-test-data/smalldata/prostate/prostate.csv.zip") >>> prostate[2] = prostate[2].asfactor() >>> prostate[4] = prostate[4].asfactor() >>> prostate[5] = prostate[5].asfactor() >>> prostate[8] = prostate[8].asfactor() >>> predictors = ["AGE","RACE","DPROS","DCAPS","PSA","VOL","GLEASON"] >>> response = "CAPSULE" >>> train, valid = prostate.split_frame(ratios=[.8],seed=1234) >>> pros_glm = H2OGeneralizedLinearEstimator(family="binomial") >>> pros_glm.train(x = predictors, ... y = response, ... training_frame = train, ... validation_frame = valid) >>> pros_glm.loglikelihood() """ return self._metric_json['loglikelihood']
[docs] def gini(self): """Gini coefficient. :examples: >>> from h2o.estimators.gbm import H2OGradientBoostingEstimator >>> cars = h2o.import_file("https://s3.amazonaws.com/h2o-public-test-data/smalldata/junit/cars_20mpg.csv") >>> cars["economy_20mpg"] = cars["economy_20mpg"].asfactor() >>> predictors = ["displacement","power","weight","acceleration","year"] >>> response = "economy_20mpg" >>> train, valid = cars.split_frame(ratios = [.8], seed = 1234) >>> cars_gbm = H2OGradientBoostingEstimator(seed = 1234) >>> cars_gbm.train(x = predictors, ... y = response, ... training_frame = train, ... validation_frame = valid) >>> cars_gbm.gini() """ return self._metric_json['Gini']
[docs] def mse(self): """The MSE for this set of metrics. :examples: >>> from h2o.estimators.gbm import H2OGradientBoostingEstimator >>> cars = h2o.import_file("https://s3.amazonaws.com/h2o-public-test-data/smalldata/junit/cars_20mpg.csv") >>> cars["economy_20mpg"] = cars["economy_20mpg"].asfactor() >>> predictors = ["displacement","power","weight","acceleration","year"] >>> response = "economy_20mpg" >>> train, valid = cars.split_frame(ratios = [.8], seed = 1234) >>> cars_gbm = H2OGradientBoostingEstimator(seed = 1234) >>> cars_gbm.train(x = predictors, ... y = response, ... training_frame = train, ... validation_frame = valid) >>> cars_gbm.mse() """ return self._metric_json['MSE']
[docs] def rmse(self): """The RMSE for this set of metrics. :examples: >>> from h2o.estimators.gbm import H2OGradientBoostingEstimator >>> cars = h2o.import_file("https://s3.amazonaws.com/h2o-public-test-data/smalldata/junit/cars_20mpg.csv") >>> cars["economy_20mpg"] = cars["economy_20mpg"].asfactor() >>> predictors = ["displacement","power","weight","acceleration","year"] >>> response = "economy_20mpg" >>> train, valid = cars.split_frame(ratios = [.8], seed = 1234) >>> cars_gbm = H2OGradientBoostingEstimator(seed = 1234) >>> cars_gbm.train(x = predictors, ... y = response, ... training_frame = train, ... validation_frame = valid) >>> cars_gbm.rmse() """ return self._metric_json['RMSE']
[docs] def mae(self): """The MAE for this set of metrics. :examples: >>> from h2o.estimators.gbm import H2OGradientBoostingEstimator >>> cars = h2o.import_file("https://s3.amazonaws.com/h2o-public-test-data/smalldata/junit/cars_20mpg.csv") >>> predictors = ["displacement","power","weight","acceleration","year"] >>> response = "cylinders" >>> train, valid = cars.split_frame(ratios = [.8], seed = 1234) >>> cars_gbm = H2OGradientBoostingEstimator(distribution = "poisson", ... seed = 1234) >>> cars_gbm.train(x = predictors, ... y = response, ... training_frame = train, ... validation_frame = valid) >>> cars_gbm.mae() """ return self._metric_json['mae']
[docs] def rmsle(self): """The RMSLE for this set of metrics. :examples: >>> from h2o.estimators.gbm import H2OGradientBoostingEstimator >>> cars = h2o.import_file("https://s3.amazonaws.com/h2o-public-test-data/smalldata/junit/cars_20mpg.csv") >>> predictors = ["displacement","power","weight","acceleration","year"] >>> response = "cylinders" >>> train, valid = cars.split_frame(ratios = [.8], seed = 1234) >>> cars_gbm = H2OGradientBoostingEstimator(distribution = "poisson", ... seed = 1234) >>> cars_gbm.train(x = predictors, ... y = response, ... training_frame = train, ... validation_frame = valid) >>> cars_gbm.rmsle() """ return self._metric_json['rmsle']
[docs] def residual_deviance(self): """The residual deviance if the model has it, otherwise None. :examples: >>> from h2o.estimators.glm import H2OGeneralizedLinearEstimator >>> prostate = h2o.import_file("http://s3.amazonaws.com/h2o-public-test-data/smalldata/prostate/prostate.csv.zip") >>> prostate[2] = prostate[2].asfactor() >>> prostate[4] = prostate[4].asfactor() >>> prostate[5] = prostate[5].asfactor() >>> prostate[8] = prostate[8].asfactor() >>> predictors = ["AGE","RACE","DPROS","DCAPS","PSA","VOL","GLEASON"] >>> response = "CAPSULE" >>> train, valid = prostate.split_frame(ratios=[.8],seed=1234) >>> pros_glm = H2OGeneralizedLinearEstimator(family="binomial") >>> pros_glm.train(x = predictors, ... y = response, ... training_frame = train, ... validation_frame = valid) >>> pros_glm.residual_deviance() """ if MetricsBase._has(self._metric_json, "residual_deviance"): return self._metric_json["residual_deviance"] return None
[docs] def hglm_metric(self, metric_string): if MetricsBase._has(self._metric_json, metric_string): return self._metric_json[metric_string] return None
[docs] def residual_degrees_of_freedom(self): """The residual DoF if the model has residual deviance, otherwise None. :examples: >>> from h2o.estimators.glm import H2OGeneralizedLinearEstimator >>> prostate = h2o.import_file("http://s3.amazonaws.com/h2o-public-test-data/smalldata/prostate/prostate.csv.zip") >>> prostate[2] = prostate[2].asfactor() >>> prostate[4] = prostate[4].asfactor() >>> prostate[5] = prostate[5].asfactor() >>> prostate[8] = prostate[8].asfactor() >>> predictors = ["AGE","RACE","DPROS","DCAPS","PSA","VOL","GLEASON"] >>> response = "CAPSULE" >>> train, valid = prostate.split_frame(ratios=[.8],seed=1234) >>> pros_glm = H2OGeneralizedLinearEstimator(family="binomial") >>> pros_glm.train(x = predictors, ... y = response, ... training_frame = train, ... validation_frame = valid) >>> pros_glm.residual_degrees_of_freedom() """ if MetricsBase._has(self._metric_json, "residual_degrees_of_freedom"): return self._metric_json["residual_degrees_of_freedom"] return None
[docs] def null_deviance(self): """The null deviance if the model has residual deviance, otherwise None. :examples: >>> from h2o.estimators.glm import H2OGeneralizedLinearEstimator >>> prostate = h2o.import_file("http://s3.amazonaws.com/h2o-public-test-data/smalldata/prostate/prostate.csv.zip") >>> prostate[2] = prostate[2].asfactor() >>> prostate[4] = prostate[4].asfactor() >>> prostate[5] = prostate[5].asfactor() >>> prostate[8] = prostate[8].asfactor() >>> predictors = ["AGE","RACE","DPROS","DCAPS","PSA","VOL","GLEASON"] >>> response = "CAPSULE" >>> train, valid = prostate.split_frame(ratios=[.8],seed=1234) >>> pros_glm = H2OGeneralizedLinearEstimator(family="binomial") >>> pros_glm.train(x = predictors, ... y = response, ... training_frame = train, ... validation_frame = valid) >>> pros_glm.null_deviance() """ if MetricsBase._has(self._metric_json, "null_deviance"): return self._metric_json["null_deviance"] return None
[docs] def null_degrees_of_freedom(self): """The null DoF if the model has residual deviance, otherwise None. :examples: >>> from h2o.estimators.glm import H2OGeneralizedLinearEstimator >>> prostate = h2o.import_file("http://s3.amazonaws.com/h2o-public-test-data/smalldata/prostate/prostate.csv.zip") >>> prostate[2] = prostate[2].asfactor() >>> prostate[4] = prostate[4].asfactor() >>> prostate[5] = prostate[5].asfactor() >>> prostate[8] = prostate[8].asfactor() >>> predictors = ["AGE","RACE","DPROS","DCAPS","PSA","VOL","GLEASON"] >>> response = "CAPSULE" >>> train, valid = prostate.split_frame(ratios=[.8],seed=1234) >>> pros_glm = H2OGeneralizedLinearEstimator(family="binomial") >>> pros_glm.train(x = predictors, ... y = response, ... training_frame = train, ... validation_frame = valid) >>> pros_glm.null_degrees_of_freedom() """ if MetricsBase._has(self._metric_json, "null_degrees_of_freedom"): return self._metric_json["null_degrees_of_freedom"] return None
# private accessor for mean per-class error - the public version is overridden in H2OBinomialModelMetrics with # a method with different return semantics def _mean_per_class_error(self): return self._metric_json['mean_per_class_error']
[docs] def mean_per_class_error(self): """The mean per class error. :examples: >>> from h2o.estimators.glm import H2OGeneralizedLinearEstimator >>> prostate = h2o.import_file("http://s3.amazonaws.com/h2o-public-test-data/smalldata/prostate/prostate.csv.zip") >>> prostate[2] = prostate[2].asfactor() >>> prostate[4] = prostate[4].asfactor() >>> prostate[5] = prostate[5].asfactor() >>> prostate[8] = prostate[8].asfactor() >>> predictors = ["AGE","RACE","DPROS","DCAPS","PSA","VOL","GLEASON"] >>> response = "CAPSULE" >>> train, valid = prostate.split_frame(ratios=[.8],seed=1234) >>> pros_glm = H2OGeneralizedLinearEstimator(family="binomial") >>> pros_glm.train(x = predictors, ... y = response, ... training_frame = train, ... validation_frame = valid) >>> pros_glm.mean_per_class_error() """ return self._mean_per_class_error()
[docs] def custom_metric_name(self): """Name of custom metric or None.""" if MetricsBase._has(self._metric_json, "custom_metric_name"): return self._metric_json['custom_metric_name'] else: return None
[docs] def custom_metric_value(self): """Value of custom metric or None.""" if MetricsBase._has(self._metric_json, "custom_metric_value"): return self._metric_json['custom_metric_value'] else: return None