Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

#7332 SparkUI - saving and loading configuration profiles #7528

Merged
merged 10 commits into from
Jun 19, 2018
Merged

#7332 SparkUI - saving and loading configuration profiles #7528

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
3016fb7
SparkUI - saving and loading configuration profiles
Jun 13, 2018
c608461
SparkUI - changed config structure, restored default config
Jun 15, 2018
e0b0361
#7332 SparkUI - pr changes:
Jun 18, 2018
8c3a677
#7332 SparkUI - bugfix for config structure
Jun 19, 2018
aadb86e
#7332 changed default config name to empty string
Jun 19, 2018
43c5b71
Merge remote-tracking branch 'remotes/origin/master' into lmitusinski…
Jun 19, 2018
d403566
#7332 SparkUI - fix for changing profile value after stopping session
Jun 19, 2018
8e6b4f6
#7332 SparkUI - disable/enable form on start/stop spark session
Jun 19, 2018
6acfa4a
#7332 SparkUI - fixed saving default parameters
Jun 19, 2018
210e970
#7332 SparkUI - changed profile config structure
Jun 19, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
13 changes: 12 additions & 1 deletion js/notebook/src/shared/style/spark.scss
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,11 @@
}
}

.bx-spark-save-button {
margin-left: 10px;
width: 80px;
}

.bx-toolbar-spark-widget {
display: inline-block;
overflow: hidden;
Expand Down Expand Up @@ -326,7 +331,7 @@
-webkit-transform: rotate(300deg);
transform: rotate(300deg);
-webkit-animation-delay: -0.083333333333333s;
animation-delay: -0.083333333333333s;
animation-delay: -0.083333333333AAA333s;
}
&:nth-child(12) {
-webkit-transform: rotate(330deg);
Expand All @@ -336,3 +341,9 @@
}
}
}

.bx-spark-profile {
.widget-label {
width: 140px !important
}
}
8 changes: 8 additions & 0 deletions kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/PropertiesWidget.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,4 +39,12 @@ public VBox getWidget() {
public void add(PropertyItem propertyItem) {
this.widget.add(propertyItem);
}

public void disable() {
getItems().forEach(x -> x.disable());
}

public void enable() {
getItems().forEach(x -> x.enable());
}
}
12 changes: 12 additions & 0 deletions kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/PropertyItem.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,16 @@ public String getNameAsString() {
public String getValueAsString() {
return value.getValue();
}

public void disable() {
name.setDisabled(true);
value.setDisabled(true);
remove.setDisabled(true);
}

public void enable() {
name.setDisabled(false);
value.setDisabled(false);
remove.setDisabled(false);
}
}
17 changes: 17 additions & 0 deletions kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,23 @@ public List<Configuration> getConfiguration() {
.collect(Collectors.toList());
}

public void setConfiguration(Map<String, String> advancedSettings) {
List<PropertyItem> propertyItems = createPropertyItems(advancedSettings);
this.properties = new PropertiesWidget(propertyItems);
this.remove(this.getChildren().get(0));
add(new VBox(asList(this.header, this.properties.getWidget())));
}

public void setDisabledToAll() {
this.add.setDisabled(true);
this.properties.disable();
}

public void setEnabledToAll() {
this.add.setDisabled(false);
this.properties.enable();
}

public static class Configuration {

private String name;
Expand Down
3 changes: 3 additions & 0 deletions kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkEngineImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,9 @@ private SparkConf createSparkConf(List<SparkConfiguration.Configuration> configu
SparkConf sparkConf = new SparkConf();
sparkConf.set(SPARK_EXTRA_LISTENERS, old.get(SPARK_EXTRA_LISTENERS));
sparkConf.set(BEAKERX_ID, old.get(BEAKERX_ID));
if (old.contains(SPARK_APP_NAME)) {
sparkConf.set(SPARK_APP_NAME, old.get(SPARK_APP_NAME));
}
configurations.forEach(x -> {
if (x.getName() != null) {
sparkConf.set(x.getName(), (x.getValue() != null) ? x.getValue() : "");
Expand Down
16 changes: 9 additions & 7 deletions kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUI.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import com.twosigma.beakerx.kernel.KernelManager;
import com.twosigma.beakerx.kernel.msg.StacktraceHtmlPrinter;
import com.twosigma.beakerx.message.Message;
import org.apache.spark.SparkConf;
import org.apache.spark.sql.SparkSession;

import java.util.ArrayList;
Expand All @@ -40,6 +39,10 @@ public class SparkUI extends VBox implements SparkUIApi {
public static final String SPARK_MASTER_DEFAULT = "local[*]";
private static final String SPARK_APP_ID = "sparkAppId";
public static final String ERROR_CREATING_SPARK_SESSION = "Error creating SparkSession, see the console log for more explanation";
public static final String SPARK_EXECUTOR_CORES_DEFAULT = "10";
public static final String SPARK_EXECUTOR_MEMORY_DEFAULT = "8g";
public static final Map<String, String> SPARK_ADVANCED_OPTIONS_DEFAULT = new HashMap<>();


private final SparkUIForm sparkUIForm;
private VBox sparkUIFormPanel;
Expand All @@ -60,7 +63,7 @@ public class SparkUI extends VBox implements SparkUIApi {
this.sparkUIFormPanel = new VBox(new ArrayList<>());
add(sparkUIFormPanel);
SparkVariable.putSparkUI(this);
this.sparkUIForm = new SparkUIForm(sparkEngine, this::initSparkContext);
this.sparkUIForm = new SparkUIForm(sparkEngine, sparkUiDefaults, this::initSparkContext);
this.sparkUIFormPanel.add(sparkUIForm);
}

Expand Down Expand Up @@ -113,7 +116,8 @@ private void configureSparkContext(Message parentMessage, KernelFunctionality ke
this.sparkUIForm.sendError(StacktraceHtmlPrinter.printRedBold(ERROR_CREATING_SPARK_SESSION));
} else {
singleSparkSession.active();
saveSparkConf(sparkEngine.getSparkConf());
sparkUIForm.saveDefaults();
sparkUiDefaults.saveProfileName(sparkUIForm.getProfileName());
applicationStart();
}
} catch (Exception e) {
Expand All @@ -128,6 +132,7 @@ private SparkSession getSparkSession() {
private void applicationStart() {
this.statusPanel = new SparkUIStatus(message -> getSparkSession().sparkContext().stop());
this.sparkUIForm.setDomClasses(new ArrayList<>(asList("bx-disabled")));
this.sparkUIForm.setAllToDisabled();
add(0, this.statusPanel);
sendUpdate(SPARK_APP_ID, sparkEngine.getSparkAppId());
sendUpdate("sparkUiWebUrl", sparkEngine.getSparkUiWebUrl());
Expand All @@ -137,6 +142,7 @@ private void applicationStart() {
@Override
public void applicationEnd() {
this.sparkUIForm.setDomClasses(new ArrayList<>());
this.sparkUIForm.setAllToEnabled();
removeStatusPanel();
singleSparkSession.inActive();
}
Expand Down Expand Up @@ -245,10 +251,6 @@ public Button getConnectButton() {
return this.sparkUIForm.getConnectButton();
}

void saveSparkConf(SparkConf sparkConf) {
sparkUiDefaults.saveSparkConf(sparkConf);
}

public interface SparkUIFactory {
SparkUI create(SparkSession.Builder builder);
}
Expand Down
1 change: 1 addition & 0 deletions kernel/sparkex/src/main/java/com/twosigma/beakerx/widget/SparkUIApi.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ public interface SparkUIApi {
String SPARK_EXECUTOR_MEMORY = "spark.executor.memory";
String SPARK_EXECUTOR_CORES = "spark.executor.cores";
String SPARK_EXTRA_LISTENERS = "spark.extraListeners";
String SPARK_ADVANCED_OPTIONS = "properties";
String BEAKERX_ID = "beakerx.id";
List<String> STANDARD_SETTINGS = Arrays.asList(SPARK_MASTER, SPARK_EXECUTOR_MEMORY, SPARK_EXECUTOR_CORES, SPARK_APP_NAME, BEAKERX_ID, SPARK_EXTRA_LISTENERS, SPARK_REPL_CLASS_OUTPUT_DIR);
String SPARK_SESSION_NAME = "spark";
Expand Down