H2OContext

Abstract Value Members

abstract def downloadH2OLogs(destinationDir: String, logContainer: String = "ZIP"): String
abstract def getFlowEndpoint(): String

Attributes
protected
abstract def getH2OBuildInfo(nodes: Array[NodeDesc]): H2OBuildInfo

Attributes
protected
abstract def getH2OClusterInfo(nodes: Array[NodeDesc]): H2OClusterInfo

Attributes
protected
abstract def getH2OEndpointIp(): String

Attributes
protected
abstract def getH2OEndpointPort(): Int

Attributes
protected
abstract def getH2ONodes(): Array[NodeDesc]
abstract def getSelfNodeDesc(): Option[NodeDesc]

Attributes
protected
abstract def initBackend(): Unit

Attributes
protected
abstract def setTimeZoneOfH2OCluster(timeZone: String): Unit

Attributes
protected

Concrete Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def asDataFrame(s: String): DataFrame
def asDataFrame(s: String, copyMetadata: Boolean): DataFrame
def asDataFrame[T <: Frame](fr: T, copyMetadata: Boolean = true): DataFrame

Convert given H2O frame into DataFrame type
def asH2OFrame(fr: Frame): H2OFrame

Create a new H2OFrame based on existing Frame
def asH2OFrame(s: String): H2OFrame

Create a new H2OFrame based on existing Frame referenced by its key.
def asH2OFrame[T <: Product](ds: Dataset[T], frameName: String)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): H2OFrame
def asH2OFrame[T <: Product](ds: Dataset[T], frameName: Option[String])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): H2OFrame
def asH2OFrame[T <: Product](ds: Dataset[T])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): H2OFrame

Transforms Dataset[Supported type] to H2OFrame
def asH2OFrame(df: DataFrame, frameName: String): H2OFrame
def asH2OFrame(df: DataFrame, frameName: Option[String]): H2OFrame
def asH2OFrame(df: DataFrame): H2OFrame

Transform DataFrame to H2OFrame
def asH2OFrame(rdd: SupportedRDD, frameName: String): H2OFrame
def asH2OFrame(rdd: SupportedRDD, frameName: Option[String]): H2OFrame
def asH2OFrame(rdd: SupportedRDD): H2OFrame

Transforms RDD[Supported type] to H2OFrame
def asH2OFrameKeyString(df: DataFrame, frameName: Option[String]): String
def asH2OFrameKeyString(df: DataFrame, frameName: String): String
def asH2OFrameKeyString(df: DataFrame): String

The method transforms Spark DataFrame to H2OFrame and returns String representation of its key.
The method transforms Spark DataFrame to H2OFrame and returns String representation of its key.
df
Input data frame
returns
String representation of H2O Frame Key
def asH2OFrameKeyString(rdd: SupportedRDD, frameName: Option[String]): String
def asH2OFrameKeyString(rdd: SupportedRDD, frameName: String): String
def asH2OFrameKeyString(rdd: SupportedRDD): String

The method transforms RDD to H2OFrame and returns String representation of its key.
The method transforms RDD to H2OFrame and returns String representation of its key.
rdd
Input RDD
returns
String representation of H2O Frame Key
final def asInstanceOf[T0]: T0

Definition Classes
Any
def asRDD[A <: Product](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A], arg1: ClassTag[A]): AnyRef { def apply[T <: org.apache.spark.h2o.Frame](fr: T): org.apache.spark.h2o.RDD[A] }

A generic convert of Frame into Product RDD type
A generic convert of Frame into Product RDD type
Consider using asH2OFrame since asRDD has several limitations such as that asRDD can't be used in Spark REPL in case we are RDD[T] where T is class defined in REPL. This is because class T is created as inner class and we are not able to create instance of class T without outer scope - which is impossible to get.
This code: hc.asRDD[PUBDEV458Type](rdd) will need to be call as hc.asRDD[PUBDEV458Type].apply(rdd)
def asRDD[A <: Product](fr: H2OFrame)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A], arg1: ClassTag[A]): RDD[A]

Convert given H2O frame into a Product RDD type
Convert given H2O frame into a Product RDD type
Consider using asH2OFrame since asRDD has several limitations such as that asRDD can't be used in Spark REPL in case we are RDD[T] where T is class defined in REPL. This is because class T is created as inner class and we are not able to create instance of class T without outer scope - which is impossible to get.
val backend: SparklingBackend

Used backend
Used backend

Attributes
protected
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def flowURL(): String
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getConf: H2OConf

Return a copy of this H2OContext's configuration.
Return a copy of this H2OContext's configuration. The configuration cannot be changed at runtime.
def getSparklingWaterHeartBeatEvent(): SparklingWaterHeartbeatEvent

Attributes
protected
def h2oLocalClient: String

Returns location of REST API of H2O client
def h2oLocalClientIp: String

Returns IP of H2O client
def h2oLocalClientPort: Int

Returns port where H2O REST API is exposed
def hashCode(): Int

Definition Classes
AnyRef → Any
object implicits extends H2OContextImplicits with Serializable

Define implicits available via h2oContext.implicits._
def importHiveTable(database: String = HiveTableImporter.DEFAULT_DATABASE, table: String, partitions: Array[Array[String]] = null, allowMultiFormat: Boolean = false): water.fvec.Frame

Definition Classes
H2OContextUtils
def init(): H2OContext

This method connects to external H2O cluster if spark.ext.h2o.externalClusterMode is set to true, otherwise it creates new H2O cluster living in Spark
This method connects to external H2O cluster if spark.ext.h2o.externalClusterMode is set to true, otherwise it creates new H2O cluster living in Spark

Attributes
protected
def initializeLogIfNecessary(isInterpreter: Boolean): Unit

Attributes
protected
Definition Classes
Logging
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isRunningOnCorrectSpark(sc: SparkContext): Boolean

Checks whether version of provided Spark is the same as Spark's version designated for this Sparkling Water version.
Checks whether version of provided Spark is the same as Spark's version designated for this Sparkling Water version. We check for correct version in shell scripts and during the build but we need to do the check also in the code in cases when the user executes for example spark-shell command with sparkling water assembly jar passed as --jars and initiates H2OContext. (Because in that case no check for correct Spark version has been done so far.)

Definition Classes
H2OContextUtils
def isTesting(sc: SparkContext): Boolean

Return true if running inside spark/sparkling water test.
Return true if running inside spark/sparkling water test.
sc
Spark Context
returns
true if the actual run is test run

Definition Classes
H2OContextUtils
def isTraceEnabled(): Boolean

Attributes
protected
Definition Classes
Logging
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logName: String

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def openFlow(): Unit

Open H2O Flow running in this client.
def openURI(sc: SparkContext, uri: String): Unit

Open browser for given address.
Open browser for given address.
uri
address to open in browser, e.g., http://example.com

Definition Classes
H2OContextUtils
def setH2OClientLogLevel(level: String): Unit

Set log level for the client running in driver
def setH2OLogLevel(level: String): Unit

Set H2O log level for the client and all executors
def setH2ONodeLogLevel(level: String): Unit

Set log level for all H2O services running on executors
val sparkContext: SparkContext
val sparkSession: SparkSession
def stop(stopSparkContext: Boolean = false): Unit

Stops H2O context.
Stops H2O context.
stopSparkContext
stop also spark context
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toH2OFrameKey[T <: Product](ds: Dataset[T], frameName: String)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): Key[Frame]
def toH2OFrameKey[T <: Product](ds: Dataset[T], frameName: Option[String])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): Key[Frame]
def toH2OFrameKey[T <: Product](ds: Dataset[T])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): Key[Frame]

Transforms Dataset[Supported type] to H2OFrame key
def toH2OFrameKey(df: DataFrame, frameName: String): Key[Frame]
def toH2OFrameKey(df: DataFrame, frameName: Option[String]): Key[Frame]
def toH2OFrameKey(df: DataFrame): Key[Frame]

Transform DataFrame to H2OFrame key
def toH2OFrameKey(rdd: SupportedRDD, frameName: String): Key[_]
def toH2OFrameKey(rdd: SupportedRDD, frameName: Option[String]): Key[_]
def toH2OFrameKey(rdd: SupportedRDD): Key[_]

Transforms RDD[Supported type] to H2OFrame key
def toString(): String

Definition Classes
H2OContext → AnyRef → Any
def verifyLogContainer(logContainer: String): Unit

Attributes
protected
Definition Classes
H2OContextUtils
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
def withConversionDebugPrints[R <: water.fvec.Frame](sc: SparkContext, conversionName: String, block: ⇒ R): R

Definition Classes
H2OContextUtils

Related Docs: object H2OContext | package h2o

abstract class H2OContext extends Logging with H2OContextUtils

Abstract Value Members

abstract def downloadH2OLogs(destinationDir: String, logContainer: String = "ZIP"): String

abstract def getFlowEndpoint(): String

abstract def getH2OBuildInfo(nodes: Array[NodeDesc]): H2OBuildInfo

abstract def getH2OClusterInfo(nodes: Array[NodeDesc]): H2OClusterInfo

abstract def getH2OEndpointIp(): String

abstract def getH2OEndpointPort(): Int

abstract def getH2ONodes(): Array[NodeDesc]

abstract def getSelfNodeDesc(): Option[NodeDesc]

abstract def initBackend(): Unit

abstract def setTimeZoneOfH2OCluster(timeZone: String): Unit

Concrete Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def asDataFrame(s: String): DataFrame

def asDataFrame(s: String, copyMetadata: Boolean): DataFrame

def asDataFrame[T <: Frame](fr: T, copyMetadata: Boolean = true): DataFrame

def asH2OFrame(fr: Frame): H2OFrame

def asH2OFrame(s: String): H2OFrame

def asH2OFrame[T <: Product](ds: Dataset[T], frameName: String)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): H2OFrame

def asH2OFrame[T <: Product](ds: Dataset[T], frameName: Option[String])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): H2OFrame

def asH2OFrame[T <: Product](ds: Dataset[T])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): H2OFrame

def asH2OFrame(df: DataFrame, frameName: String): H2OFrame

def asH2OFrame(df: DataFrame, frameName: Option[String]): H2OFrame

def asH2OFrame(df: DataFrame): H2OFrame

def asH2OFrame(rdd: SupportedRDD, frameName: String): H2OFrame

def asH2OFrame(rdd: SupportedRDD, frameName: Option[String]): H2OFrame

def asH2OFrame(rdd: SupportedRDD): H2OFrame

def asH2OFrameKeyString(df: DataFrame, frameName: Option[String]): String

def asH2OFrameKeyString(df: DataFrame, frameName: String): String

def asH2OFrameKeyString(df: DataFrame): String

def asH2OFrameKeyString(rdd: SupportedRDD, frameName: Option[String]): String

def asH2OFrameKeyString(rdd: SupportedRDD, frameName: String): String

def asH2OFrameKeyString(rdd: SupportedRDD): String

final def asInstanceOf[T0]: T0

def asRDD[A <: Product](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A], arg1: ClassTag[A]): AnyRef { def apply[T <: org.apache.spark.h2o.Frame](fr: T): org.apache.spark.h2o.RDD[A] }

def asRDD[A <: Product](fr: H2OFrame)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A], arg1: ClassTag[A]): RDD[A]

val backend: SparklingBackend

def clone(): AnyRef

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

def flowURL(): String

final def getClass(): Class[_]

def getConf: H2OConf

def getSparklingWaterHeartBeatEvent(): SparklingWaterHeartbeatEvent

def h2oLocalClient: String

def h2oLocalClientIp: String

def h2oLocalClientPort: Int

def hashCode(): Int

object implicits extends H2OContextImplicits with Serializable

def importHiveTable(database: String = HiveTableImporter.DEFAULT_DATABASE, table: String, partitions: Array[Array[String]] = null, allowMultiFormat: Boolean = false): water.fvec.Frame

def init(): H2OContext

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

final def isInstanceOf[T0]: Boolean

def isRunningOnCorrectSpark(sc: SparkContext): Boolean

def isTesting(sc: SparkContext): Boolean

def isTraceEnabled(): Boolean

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def openFlow(): Unit

def openURI(sc: SparkContext, uri: String): Unit

def setH2OClientLogLevel(level: String): Unit