-
Notifications
You must be signed in to change notification settings - Fork 202
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[Part 1]: Support for hot task executors in Mantis #145
Changes from all commits
d1e0162
e22e6e8
6ca0440
50113e8
cadf9ef
1bf6737
a172420
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1 +1 @@ | ||
* @codyrioux @neerajrj @nickmahilani @jeffchao @piygoyal @calvin681 @sundargates @Andyz26 | ||
* @codyrioux @nickmahilani @jeffchao @piygoyal @calvin681 @sundargates @Andyz26 @hmitnflx |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
lombok.anyConstructor.addConstructorProperties = true |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
/* | ||
* Copyright 2022 Netflix, Inc. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
package io.mantisrx.common; | ||
|
||
import java.io.Serializable; | ||
import lombok.Value; | ||
|
||
/** | ||
* Ack is sent whenever an effect has taken place on the side of the receiver. | ||
* Ack instance supports both JSON / Java serialization. | ||
*/ | ||
@Value(staticConstructor = "getInstance") | ||
public class Ack implements Serializable { | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,53 @@ | ||
/* | ||
* Copyright 2022 Netflix, Inc. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
package io.mantisrx.common; | ||
|
||
import io.mantisrx.shaded.com.fasterxml.jackson.core.type.TypeReference; | ||
import io.mantisrx.shaded.com.fasterxml.jackson.databind.DeserializationFeature; | ||
import io.mantisrx.shaded.com.fasterxml.jackson.databind.ObjectMapper; | ||
import io.mantisrx.shaded.com.fasterxml.jackson.databind.SerializationFeature; | ||
import io.mantisrx.shaded.com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider; | ||
import io.mantisrx.shaded.com.fasterxml.jackson.datatype.jdk8.Jdk8Module; | ||
import java.io.IOException; | ||
import org.slf4j.Logger; | ||
import org.slf4j.LoggerFactory; | ||
|
||
public class JsonSerializer { | ||
private static final Logger logger = LoggerFactory.getLogger(JsonSerializer.class); | ||
|
||
private static final ObjectMapper defaultObjectMapper = new ObjectMapper() | ||
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) | ||
.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false) | ||
.registerModule(new Jdk8Module()); | ||
public static final SimpleFilterProvider DEFAULT_FILTER_PROVIDER; | ||
|
||
static { | ||
DEFAULT_FILTER_PROVIDER = new SimpleFilterProvider(); | ||
DEFAULT_FILTER_PROVIDER.setFailOnUnknownId(false); | ||
} | ||
|
||
public <T> T fromJSON(String json, Class<T> expectedType) throws IOException { | ||
return defaultObjectMapper.readerFor(expectedType).readValue(json); | ||
} | ||
|
||
public <T> T fromJSON(String json, TypeReference<T> expectedType) throws IOException { | ||
return defaultObjectMapper.readerFor(expectedType).readValue(json); | ||
} | ||
|
||
public String toJson(Object object) throws IOException { | ||
return defaultObjectMapper.writeValueAsString(object); | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
/* | ||
* Copyright 2022 Netflix, Inc. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package io.mantisrx.common; | ||
|
||
import static org.junit.Assert.assertEquals; | ||
|
||
import org.junit.Test; | ||
|
||
public class AckTest { | ||
private final JsonSerializer serializer = new JsonSerializer(); | ||
|
||
@Test | ||
public void testAckInstance() throws Exception { | ||
Ack ack = Ack.getInstance(); | ||
String serialized = serializer.toJson(ack); | ||
Ack actual = serializer.fromJSON(serialized, Ack.class); | ||
assertEquals(ack, actual); | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
/* | ||
* Copyright 2022 Netflix, Inc. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
package io.mantisrx.server.master.resourcecluster; | ||
|
||
import lombok.RequiredArgsConstructor; | ||
import lombok.Value; | ||
|
||
/** | ||
* Represents the cluster that the task executor belongs to. | ||
*/ | ||
@RequiredArgsConstructor(staticName="of") | ||
@Value | ||
public class ClusterID { | ||
String resourceID; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this should be private? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. value classes by default in Lombok treat the fields as "private final". Here's the documentation: https://projectlombok.org/features/Value |
||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,57 @@ | ||
/* | ||
* Copyright 2022 Netflix, Inc. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
package io.mantisrx.server.master.resourcecluster; | ||
|
||
import io.mantisrx.common.Ack; | ||
import java.util.concurrent.CompletableFuture; | ||
|
||
/** | ||
* Gateway for performing all actions corresponding to the resource cluster from the task executor. | ||
*/ | ||
public interface ResourceClusterGateway { | ||
/** | ||
* triggered at the start of a task executor or when the task executor has lost connection to the resource cluster | ||
* previously | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: extra word? |
||
*/ | ||
CompletableFuture<Ack> registerTaskExecutor(TaskExecutorRegistration registration); | ||
|
||
/** | ||
* Triggered when the task executor needs to send a heartbeat every epoch after registration. | ||
* Absence of a heartbeat from the task executor implies loss of the task executor or a network partition. | ||
*/ | ||
CompletableFuture<Ack> heartBeatFromTaskExecutor(TaskExecutorHeartbeat heartbeat); | ||
|
||
/** | ||
* Triggered whenever the task executor gets occupied with a worker request or is available to do some work | ||
*/ | ||
CompletableFuture<Ack> notifyTaskExecutorStatusChange(TaskExecutorStatusChange taskExecutorStatusChange); | ||
|
||
/** | ||
* Triggered by the task executor when it's about to shut itself down. | ||
*/ | ||
CompletableFuture<Ack> disconnectTaskExecutor(TaskExecutorDisconnection taskExecutorDisconnection); | ||
|
||
/** | ||
* Exception thrown by the resource cluster whenever anyone of the state transitions of the task executor are invalid. | ||
* Note that the exception generally gets wrapped inside {@link java.util.concurrent.CompletionException}. | ||
*/ | ||
class InvalidStateTransitionException extends Exception { | ||
|
||
public InvalidStateTransitionException(String message) { | ||
super(message); | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
/* | ||
* Copyright 2022 Netflix, Inc. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package io.mantisrx.server.master.resourcecluster; | ||
|
||
import lombok.Value; | ||
|
||
/** | ||
* Datastructure representing the task executor that needs to be disconnected from the resource cluster. | ||
*/ | ||
@Value | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. constructor? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. value annotation creates an "allargsconstructor". |
||
public class TaskExecutorDisconnection { | ||
TaskExecutorID taskExecutorID; | ||
|
||
ClusterID clusterID; | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
/* | ||
* Copyright 2022 Netflix, Inc. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package io.mantisrx.server.master.resourcecluster; | ||
|
||
import lombok.Value; | ||
|
||
@Value | ||
public class TaskExecutorHeartbeat { | ||
TaskExecutorID taskExecutorID; | ||
|
||
ClusterID clusterID; | ||
|
||
TaskExecutorReport taskExecutorReport; | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
/* | ||
* Copyright 2022 Netflix, Inc. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
package io.mantisrx.server.master.resourcecluster; | ||
|
||
import java.util.UUID; | ||
import lombok.RequiredArgsConstructor; | ||
import lombok.Value; | ||
|
||
/** | ||
* ID of the task executor. Reason this is wrapped inside a type is to have strong typing when dealing | ||
* with different types of IDs within the system. | ||
*/ | ||
@RequiredArgsConstructor(staticName="of") | ||
@Value | ||
public class TaskExecutorID { | ||
String resourceId; | ||
|
||
public static TaskExecutorID generate() { | ||
return new TaskExecutorID(UUID.randomUUID().toString()); | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
/* | ||
* Copyright 2022 Netflix, Inc. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
package io.mantisrx.server.master.resourcecluster; | ||
|
||
import io.mantisrx.common.WorkerPorts; | ||
import io.mantisrx.runtime.MachineDefinition; | ||
import lombok.Value; | ||
|
||
/** | ||
* Data structure used at the time of registration by the task executor. | ||
* Different fields help identify the task executor in different dimensions. | ||
*/ | ||
@Value | ||
public class TaskExecutorRegistration { | ||
TaskExecutorID taskExecutorID; | ||
|
||
ClusterID clusterID; | ||
|
||
// RPC address that's used to talk to the task executor | ||
String taskExecutorAddress; | ||
|
||
// host name of the task executor | ||
String hostname; | ||
|
||
// ports used by the task executor for various purposes. | ||
WorkerPorts workerPorts; | ||
|
||
// machine information identifies the cpu/mem/disk/network resources of the task executor. | ||
MachineDefinition machineDefinition; | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit: I'd call out the semantic difference between this and
static toJson
in comments.Also, could we make this static?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I have removed all the static methods.