array_contains

ArrayContainsTransformer

ArrayContainsTransformer(
    inputCol=None,
    inputCols=None,
    outputCol=None,
    inputDtype=None,
    outputDtype=None,
    layerName=None,
    mathFloatConstant=None,
)

Bases: BaseTransformer, SingleInputSingleOutputParams, MultiInputSingleOutputParams, MathFloatConstantParams

ArrayContainsLayer Spark Transformer for use in Spark pipelines.

Checks whether a scalar value is present in a (possibly nested) numeric array. The value is either a second input column (inputCols) or the mathFloatConstant (inputCol). Outputs a boolean; set outputDtype to cast the result.

Example:

df.show() +---------+ | a| +---------+ |[1, 2, 3]| |[4, 5, 6]| +---------+ t = ArrayContainsTransformer(inputCol="a", outputCol="b", mathFloatConstant=2) t.transform(df).show() +---------+-----+ | a| b| +---------+-----+ |[1, 2, 3]| true| |[4, 5, 6]|false| +---------+-----+

Initializes an ArrayContainsTransformer transformer.

Parameters:

Name	Type	Description	Default
inputCol	Optional[str]	Input array column name. Only used if inputCols is not specified. If specified, we check whether mathFloatConstant is contained in this array column.	None
inputCols	Optional[List[str]]	Input column names, given as [arrayCol, valueCol].	None
outputCol	Optional[str]	Output column name.	None
inputDtype	Optional[str]	Input data type to cast input column(s) to before transforming.	None
outputDtype	Optional[str]	Output data type to cast the output column to after transforming.	None
layerName	Optional[str]	Name of the layer. Used as the name of the Keras layer in the keras model. If not set, we use the uid of the Spark transformer.	None
mathFloatConstant	Optional[float]	Optional constant value to check for. Used with inputCol. If not provided, then inputCols is required.	None

Returns:

Type	Description
None	None - class instantiated.

Source code in src/kamae/spark/transformers/array_contains.py

@keyword_only
def __init__(
    self,
    inputCol: Optional[str] = None,
    inputCols: Optional[List[str]] = None,
    outputCol: Optional[str] = None,
    inputDtype: Optional[str] = None,
    outputDtype: Optional[str] = None,
    layerName: Optional[str] = None,
    mathFloatConstant: Optional[float] = None,
) -> None:
    """
    Initializes an ArrayContainsTransformer transformer.

    :param inputCol: Input array column name. Only used if inputCols is not
    specified. If specified, we check whether `mathFloatConstant` is contained
    in this array column.
    :param inputCols: Input column names, given as `[arrayCol, valueCol]`.
    :param outputCol: Output column name.
    :param inputDtype: Input data type to cast input column(s) to before
    transforming.
    :param outputDtype: Output data type to cast the output column to after
    transforming.
    :param layerName: Name of the layer. Used as the name of the Keras layer
    in the keras model. If not set, we use the uid of the Spark transformer.
    :param mathFloatConstant: Optional constant value to check for. Used with
    `inputCol`. If not provided, then `inputCols` is required.
    :returns: None - class instantiated.
    """
    super().__init__()
    self._setDefault(mathFloatConstant=None)
    kwargs = self._input_kwargs
    self.setParams(**kwargs)

compatible_dtypes property

compatible_dtypes

List of compatible data types for the layer. If the computation can be performed on any data type, return None.

Returns:

Type	Description
Optional[List[DataType]]	List of compatible data types for the layer.

get_keras_layer

get_keras_layer()

Gets the Keras layer for the array contains transformer.

Returns:

Type	Description
Layer	Keras layer with name equal to the layerName parameter that performs the array contains operation.

Source code in src/kamae/spark/transformers/array_contains.py

def get_keras_layer(self) -> keras.layers.Layer:
    """
    Gets the Keras layer for the array contains transformer.

    :returns: Keras layer with name equal to the layerName parameter that
    performs the array contains operation.
    """
    return ArrayContainsLayer(
        name=self.getLayerName(),
        input_dtype=self.getInputKerasDtype(),
        output_dtype=self.getOutputKerasDtype(),
        axis=-1,
        keepdims=True,
        value_constant=self.getMathFloatConstant(),
    )

setInputCols

setInputCols(value)

Sets the input columns, ensuring exactly two are provided: [arrayCol, valueCol].

Parameters:

Name	Type	Description	Default
value	List[str]	List of two input column names.	required

Returns:

Type	Description
ArrayContainsTransformer	Instance of class with input columns set.

Source code in src/kamae/spark/transformers/array_contains.py

def setInputCols(self, value: List[str]) -> "ArrayContainsTransformer":
    """
    Sets the input columns, ensuring exactly two are provided:
    `[arrayCol, valueCol]`.

    :param value: List of two input column names.
    :returns: Instance of class with input columns set.
    """
    if len(value) != 2:
        raise ValueError(f"Expected 2 input cols, received {len(value)} instead.")

    return self._set(inputCols=value)