Index

Symbols

--compute-environment sparkctl-default-config command line option --connect-server sparkctl-configure command line option --console-level sparkctl command line option --directory sparkctl-configure command line option sparkctl-default-config command line option sparkctl-start command line option sparkctl-stop command line option --driver-memory-gb sparkctl-configure command line option --dynamic-allocation sparkctl-configure command line option --executor-cores sparkctl-configure command line option --executor-memory-gb sparkctl-configure command line option --file-level sparkctl command line option --hadoop-path sparkctl-default-config command line option --history-server sparkctl-configure command line option --hive-metastore sparkctl-configure command line option --hive-tarball sparkctl-default-config command line option --local-storage sparkctl-configure command line option --metastore-dir sparkctl-configure command line option --no-connect-server sparkctl-configure command line option --no-dynamic-allocation sparkctl-configure command line option --no-history-server sparkctl-configure command line option --no-hive-metastore sparkctl-configure command line option --no-local-storage sparkctl-configure command line option --no-postgres-hive-metastore sparkctl-configure command line option --no-resource-monitor sparkctl-configure command line option --no-start sparkctl-configure command line option --no-thrift-server sparkctl-configure command line option --no-use-current-python sparkctl-configure command line option --no-wait sparkctl-start command line option --node-memory-overhead-gb sparkctl-configure command line option --postgres-hive-metastore sparkctl-configure command line option --postgresql-jar-file sparkctl-default-config command line option --python-path sparkctl-configure command line option

--reraise-exceptions sparkctl command line option --resource-monitor sparkctl-configure command line option --shuffle-partition-multiplier sparkctl-configure command line option --spark-defaults-template-file sparkctl-configure command line option --spark-log-level sparkctl-configure command line option --spark-scratch sparkctl-configure command line option --start sparkctl-configure command line option --thrift-server sparkctl-configure command line option --timeout sparkctl-start command line option --use-current-python sparkctl-configure command line option --wait sparkctl-start command line option -c sparkctl command line option -d sparkctl-configure command line option sparkctl-default-config command line option sparkctl-start command line option sparkctl-stop command line option -E sparkctl-configure command line option -e sparkctl-configure command line option sparkctl-default-config command line option -f sparkctl command line option -H sparkctl-default-config command line option -h sparkctl-default-config command line option -l sparkctl-configure command line option -M sparkctl-configure command line option -m sparkctl-configure command line option -o sparkctl-configure command line option -P sparkctl-configure command line option -p sparkctl-default-config command line option -r sparkctl command line option -s sparkctl-configure command line option -t sparkctl-configure command line option sparkctl-start command line option -w sparkctl-configure command line option

A

app (sparkctl.models.SparkConfig attribute)

B

base (sparkctl.models.RuntimeDirectories attribute)

binaries (sparkctl.models.SparkConfig attribute)

C

clean() (sparkctl.cluster_manager.ClusterManager method) clean_spark_conf_dir() (sparkctl.models.RuntimeDirectories method) ClusterManager (class in sparkctl.cluster_manager)

compute (sparkctl.models.SparkConfig attribute) ComputeEnvironment (class in sparkctl.models) configure() (sparkctl.cluster_manager.ClusterManager method)

D

directories (sparkctl.models.SparkConfig attribute)

driver_memory_gb (sparkctl.models.SparkRuntimeParams attribute)

E

enable_dynamic_allocation (sparkctl.models.SparkRuntimeParams attribute) enable_hive_metastore (sparkctl.models.SparkRuntimeParams attribute) enable_postgres_hive_metastore (sparkctl.models.SparkRuntimeParams attribute)

environment (sparkctl.models.ComputeParams attribute) executor_cores (sparkctl.models.SparkRuntimeParams attribute) executor_memory_gb (sparkctl.models.SparkRuntimeParams attribute)

F

from_config() (sparkctl.cluster_manager.ClusterManager class method)

from_config_file() (sparkctl.cluster_manager.ClusterManager class method)

G

get_events_dir() (sparkctl.models.RuntimeDirectories method) get_hive_site_file() (sparkctl.models.RuntimeDirectories method) get_spark_conf_dir() (sparkctl.models.RuntimeDirectories method) get_spark_defaults_file() (sparkctl.models.RuntimeDirectories method)

get_spark_env_file() (sparkctl.models.RuntimeDirectories method) get_spark_log_file() (sparkctl.models.RuntimeDirectories method) get_spark_session() (sparkctl.cluster_manager.ClusterManager method) get_workers() (sparkctl.cluster_manager.ClusterManager method) get_workers_file() (sparkctl.models.RuntimeDirectories method)

H

hadoop_path (sparkctl.models.BinaryLocations attribute)

hive_tarball (sparkctl.models.BinaryLocations attribute)

J

JAVA_PATH sparkctl-default-config command line option

java_path (sparkctl.models.BinaryLocations attribute)

L

load() (sparkctl.cluster_manager.ClusterManager class method)

M

make_absolute() (sparkctl.models.BinaryLocations class method) (sparkctl.models.RuntimeDirectories class method) make_default_spark_config() (in module sparkctl.config) managed_cluster() (sparkctl.cluster_manager.ClusterManager method)

metastore_dir (sparkctl.models.RuntimeDirectories attribute) module sparkctl.config sparkctl.models

N

node_memory_overhead_gb (sparkctl.models.SparkRuntimeParams attribute)

P

postgres (sparkctl.models.ComputeParams attribute) postgres_password (sparkctl.models.SparkRuntimeParams attribute)

postgresql_jar_file (sparkctl.models.BinaryLocations attribute) python_path (sparkctl.models.SparkRuntimeParams attribute)

R

resource_monitor (sparkctl.models.SparkConfig attribute)

runtime (sparkctl.models.SparkConfig attribute)

S

set_postgres_password() (sparkctl.models.SparkRuntimeParams class method) set_workers() (sparkctl.cluster_manager.ClusterManager method) shuffle_partition_multiplier (sparkctl.models.SparkRuntimeParams attribute) spark_defaults_template_file (sparkctl.models.SparkRuntimeParams attribute) spark_log_level (sparkctl.models.SparkRuntimeParams attribute) SPARK_PATH sparkctl-default-config command line option spark_path (sparkctl.models.BinaryLocations attribute) spark_scratch (sparkctl.models.RuntimeDirectories attribute) sparkctl command line option --console-level --file-level --reraise-exceptions -c -f -r sparkctl-configure command line option --connect-server --directory --driver-memory-gb --dynamic-allocation --executor-cores --executor-memory-gb --history-server --hive-metastore --local-storage --metastore-dir --no-connect-server --no-dynamic-allocation --no-history-server --no-hive-metastore --no-local-storage --no-postgres-hive-metastore --no-resource-monitor --no-start --no-thrift-server --no-use-current-python --node-memory-overhead-gb --postgres-hive-metastore --python-path --resource-monitor --shuffle-partition-multiplier --spark-defaults-template-file --spark-log-level --spark-scratch --start --thrift-server --use-current-python -d -e -E -l -M -m -o -P -s -t -w

sparkctl-default-config command line option --compute-environment --directory --hadoop-path --hive-tarball --postgresql-jar-file -d -e -H -h -p JAVA_PATH SPARK_PATH sparkctl-start command line option --directory --no-wait --timeout --wait -d -t sparkctl-stop command line option --directory -d sparkctl.config module sparkctl.models module start() (sparkctl.cluster_manager.ClusterManager method) start_connect_server (sparkctl.models.SparkRuntimeParams attribute) start_history_server (sparkctl.models.SparkRuntimeParams attribute) start_thrift_server (sparkctl.models.SparkRuntimeParams attribute) stop() (sparkctl.cluster_manager.ClusterManager method)

U

use_local_storage (sparkctl.models.SparkRuntimeParams attribute)