Skip to content

Commit

Permalink
Merge branch 'main' into new-unified-sm
Browse files Browse the repository at this point in the history
  • Loading branch information
TingDaoK committed Jul 14, 2022
2 parents d358cd4 + dd0f98a commit 868747c
Show file tree
Hide file tree
Showing 18 changed files with 236 additions and 138 deletions.
1 change: 1 addition & 0 deletions .builder/actions/localhost_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ def run(self, env):
-DrerunFailingTestsCount=5 \
-Daws.crt.memory.tracing=2 \
-Daws.crt.debugnative=true \
-Daws.crt.log.level=Error \
-Daws.crt.localhost=true"):
# Failed
actions.append("exit 1")
Expand Down
1 change: 0 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,6 @@ jobs:
- name: Configure local host
run: |
python3 -m pip install h2
ls crt/aws-c-http/
cd crt/aws-c-http/tests/py_localhost/
python3 server.py &
python3 non_tls_server.py &
Expand Down
3 changes: 2 additions & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ To send us a pull request, please:
3. Ensure local tests pass.
4. Commit to your fork using clear commit messages.
5. Send us a pull request, answering any default questions in the pull request interface.
6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation.
6. Wait for a repository collaborator to look at your PR, run it in CI, test, and review. If additional changes or discussion is needed, a contributor will get back to you, so please stay invovled in the conversation.
* Note: PRs from forks will not run in CI automatically for security reasons. If you make a PR and see that CI is left pending, this is normal and expected.

GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and
[creating a pull request](https://help.github.com/articles/creating-a-pull-request/).
Expand Down
2 changes: 1 addition & 1 deletion crt/aws-c-event-stream
2 changes: 1 addition & 1 deletion crt/aws-c-io
2 changes: 1 addition & 1 deletion crt/aws-c-mqtt
2 changes: 1 addition & 1 deletion crt/aws-c-s3
Submodule aws-c-s3 updated 37 files
+17 −4 benchmarks/README.md
+19 −2 benchmarks/benchmarks-stack/benchmarks-stack/lib/benchmarks-stack.ts
+42 −0 benchmarks/benchmarks-stack/benchmarks-stack/lib/get_p90.py
+19 −1 benchmarks/benchmarks-stack/benchmarks-stack/lib/init_instance.sh
+8 −0 benchmarks/benchmarks-stack/benchmarks-stack/lib/project_scripts/run_aws_c_s3.sh
+1 −1 benchmarks/benchmarks-stack/benchmarks-stack/package.json
+0 −13 benchmarks/dashboard-stack/lib/benchmark-config.json
+128 −3 benchmarks/dashboard-stack/lib/dashboard-stack.ts
+1 −1 benchmarks/dashboard-stack/package.json
+26 −1 include/aws/s3/private/s3_auto_ranged_put.h
+1 −1 include/aws/s3/private/s3_checksums.h
+8 −23 include/aws/s3/private/s3_list_objects.h
+124 −0 include/aws/s3/private/s3_list_parts.h
+3 −0 include/aws/s3/private/s3_meta_request_impl.h
+169 −0 include/aws/s3/private/s3_paginator.h
+16 −0 include/aws/s3/private/s3_request_messages.h
+6 −0 include/aws/s3/private/s3_util.h
+4 −0 include/aws/s3/s3.h
+25 −0 include/aws/s3/s3_client.h
+10 −8 samples/s3/main.c
+4 −0 source/s3.c
+563 −23 source/s3_auto_ranged_put.c
+2 −2 source/s3_chunk_stream.c
+96 −93 source/s3_client.c
+113 −297 source/s3_list_objects.c
+289 −0 source/s3_list_parts.c
+18 −3 source/s3_meta_request.c
+455 −0 source/s3_paginator.c
+63 −15 source/s3_request_messages.c
+10 −0 source/s3_util.c
+7 −6 tests/CMakeLists.txt
+1 −1 tests/s3_cancel_tests.c
+582 −4 tests/s3_data_plane_tests.c
+38 −10 tests/s3_test_input_stream.c
+11 −9 tests/s3_tester.c
+20 −0 tests/s3_tester.h
+57 −0 tests/s3_util_tests.c
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ public class Http2StreamManagerOptions {

private HttpClientConnectionManagerOptions connectionManagerOptions;

/* HTTP/2 specific */
private int idealConcurrentStreamsPerConnection = 100;
private boolean connectionManualWindowManagement = false;
private int maxConcurrentStreamsPerConnection = DEFAULT_MAX;

private List<Http2ConnectionSetting> initialSettingsList = new ArrayList<Http2ConnectionSetting>();

/**
Expand Down Expand Up @@ -126,6 +126,8 @@ public HttpClientConnectionManagerOptions getConnectionManagerOptions() {
}

/**
* Required.
*
* The configuration options for the connection manager under the hood.
* It controls the connection specific thing for the stream manager. See `HttpClientConnectionManagerOptions` for details.
*
Expand All @@ -146,6 +148,9 @@ public Http2StreamManagerOptions withConnectionManagerOptions(HttpClientConnecti
* Validate the stream manager options are valid to use. Throw exceptions if not.
*/
public void validateOptions() {
if(connectionManagerOptions == null) {
throw new IllegalArgumentException("Connection manager options are required.");
}
connectionManagerOptions.validateOptions();
if (maxConcurrentStreamsPerConnection <= 0) {
throw new IllegalArgumentException("Max Concurrent Streams Per Connection must be greater than zero.");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import software.amazon.awssdk.crt.CrtRuntimeException;
import software.amazon.awssdk.crt.io.ClientBootstrap;
import software.amazon.awssdk.crt.io.SocketOptions;
import software.amazon.awssdk.crt.io.TlsConnectionOptions;
import software.amazon.awssdk.crt.io.TlsContext;

/**
Expand Down Expand Up @@ -52,6 +53,8 @@ private HttpClientConnectionManager(HttpClientConnectionManagerOptions options)
SocketOptions socketOptions = options.getSocketOptions();
boolean useTls = HTTPS.equals(uri.getScheme());
TlsContext tlsContext = options.getTlsContext();
TlsConnectionOptions tlsConnectionOptions = options.getTlsConnectionOptions();

int windowSize = options.getWindowSize();
int maxConnections = options.getMaxConnections();
int port = options.getPort();
Expand Down Expand Up @@ -101,7 +104,8 @@ private HttpClientConnectionManager(HttpClientConnectionManagerOptions options)
acquireNativeHandle(httpClientConnectionManagerNew(this,
clientBootstrap.getNativeHandle(),
socketOptions.getNativeHandle(),
useTls ? tlsContext.getNativeHandle() : 0,
useTls && tlsContext!=null ? tlsContext.getNativeHandle() : 0,
useTls && tlsConnectionOptions!=null ? tlsConnectionOptions.getNativeHandle() : 0,
windowSize,
uri.getHost().getBytes(UTF8),
port,
Expand All @@ -122,7 +126,12 @@ private HttpClientConnectionManager(HttpClientConnectionManagerOptions options)
/* we don't need to add a reference to socketOptions since it's copied during connection manager construction */
addReferenceTo(clientBootstrap);
if (useTls) {
addReferenceTo(tlsContext);
if (tlsContext != null) {
addReferenceTo(tlsContext);
}
if (tlsConnectionOptions != null) {
addReferenceTo(tlsConnectionOptions);
}
}
}

Expand Down Expand Up @@ -215,6 +224,7 @@ private static native long httpClientConnectionManagerNew(HttpClientConnectionMa
long client_bootstrap,
long socketOptions,
long tlsContext,
long tlsConnectionOptions,
int windowSize,
byte[] endpoint,
int port,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import java.net.URI;
import software.amazon.awssdk.crt.io.ClientBootstrap;
import software.amazon.awssdk.crt.io.SocketOptions;
import software.amazon.awssdk.crt.io.TlsConnectionOptions;
import software.amazon.awssdk.crt.io.TlsContext;

/**
Expand All @@ -20,6 +21,7 @@ public class HttpClientConnectionManagerOptions {
private ClientBootstrap clientBootstrap;
private SocketOptions socketOptions;
private TlsContext tlsContext;
private TlsConnectionOptions tlsConnectionOptions;
private int windowSize = DEFAULT_MAX_WINDOW_SIZE;
private int bufferSize = DEFAULT_MAX_BUFFER_SIZE;
private URI uri;
Expand Down Expand Up @@ -86,6 +88,23 @@ public HttpClientConnectionManagerOptions withTlsContext(TlsContext tlsContext)
*/
public TlsContext getTlsContext() { return tlsContext; }

/**
* Sets the connection-specific TLS options to use for connections in the connection pool.
* Either TLS context or TLS connection options will be enough to set up TLS connection.
* If both set, an exception will be raised.
* @param tlsConnectionOptions The TlsConnectionOptions to use
* @return this
*/
public HttpClientConnectionManagerOptions withTlsConnectionOptions(TlsConnectionOptions tlsConnectionOptions) {
this.tlsConnectionOptions = tlsConnectionOptions;
return this;
}

/**
* @return the tls context used by connections in the connection pool
*/
public TlsConnectionOptions getTlsConnectionOptions() { return tlsConnectionOptions; }

/**
* Sets the IO channel window size to use for connections in the connection pool
* @param windowSize The initial window size to use for each connection
Expand Down Expand Up @@ -267,8 +286,12 @@ public void validateOptions() {

if (socketOptions == null) { throw new IllegalArgumentException("SocketOptions must not be null"); }

if(tlsContext!= null && tlsConnectionOptions != null) {
throw new IllegalArgumentException("Cannot set both TlsContext and TlsConnectionOptions.");
}
boolean useTls = HTTPS.equals(uri.getScheme());
if (useTls && tlsContext == null) { throw new IllegalArgumentException("TlsContext must not be null if https is used"); }
boolean tlsSet = (tlsContext!= null || tlsConnectionOptions != null);
if (useTls && !tlsSet) { throw new IllegalArgumentException("TlsContext or TlsConnectionOptions must not be null if https is used"); }

if (windowSize <= 0) { throw new IllegalArgumentException("Window Size must be greater than zero."); }

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,90 +18,52 @@ public enum TlsCipherPreference {
TLS_CIPHER_SYSTEM_DEFAULT(0),

/**
* This TlsCipherPreference contains BIKE Round 1 and SIKE Round 1 Draft Hybrid TLS Ciphers at the top of the
* preference list.
*
* For more info see:
* - https://tools.ietf.org/html/draft-campagna-tls-bike-sike-hybrid
* - https://aws.amazon.com/blogs/security/post-quantum-tls-now-supported-in-aws-kms/
*
* These Hybrid TLS ciphers perform two Key Exchanges (1 ECDHE + 1 Post-Quantum) during the TLS Handshake in order
* to combine the security of Classical ECDHE Key Exchange with the conjectured quantum-resistance of newly
* proposed key exchanges.
*
* The algorithms these new Post-Quantum ciphers are based on have been submitted to NIST's Post-Quantum Crypto
* Standardization Process, and are still under review.
*
* While these Post Quantum Hybrid TLS Ciphers are the most preferred ciphers in the preference list, classical
* ciphers are still present and can be negotiated if the TLS peer does not support these Hybrid TLS Ciphers.
*
* Since this Cipher Preference contains algorithms still being evaluated by NIST, it may stop being supported at
* any time.
* @deprecated This TlsCipherPreference is no longer supported. Use TLS_CIPHER_PREF_PQ_TLSv1_0_2021_05 instead.
*/
@Deprecated
TLS_CIPHER_KMS_PQ_TLSv1_0_2019_06(1),

/**
* This TlsCipherPreference contains SIKE Round 1 Draft Hybrid TLS Ciphers at the top of the preference list.
*
* For more info see:
* - https://tools.ietf.org/html/draft-campagna-tls-bike-sike-hybrid
* - https://aws.amazon.com/blogs/security/post-quantum-tls-now-supported-in-aws-kms/
*
* Since this Cipher Preference contains algorithms still being evaluated by NIST, it may stop being supported at
* any time.
* @deprecated This TlsCipherPreference is no longer supported. Use TLS_CIPHER_PREF_PQ_TLSv1_0_2021_05 instead.
*/
@Deprecated
TLS_CIPHER_PREF_KMS_PQ_SIKE_TLSv1_0_2019_11(2),

/**
* This TlsCipherPreference contains BIKE Round 2, SIKE Round 2, BIKE Round 1, and SIKE Round 1 Draft Hybrid TLS
* Ciphers at the top of the preference list.
*
* For more info see:
* - https://tools.ietf.org/html/draft-campagna-tls-bike-sike-hybrid
* - https://aws.amazon.com/blogs/security/post-quantum-tls-now-supported-in-aws-kms/
*
* Since this Cipher Preference contains algorithms still being evaluated by NIST, it may stop being supported at
* any time.
* @deprecated This TlsCipherPreference is no longer supported. Use TLS_CIPHER_PREF_PQ_TLSv1_0_2021_05 instead.
*/
@Deprecated
TLS_CIPHER_PREF_KMS_PQ_TLSv1_0_2020_02(3),

/**
* This TlsCipherPreference contains SIKE Round 2 and SIKE Round 1 Draft Hybrid TLS Ciphers at the top of the
* preference list.
*
* For more info see:
* - https://tools.ietf.org/html/draft-campagna-tls-bike-sike-hybrid
* - https://aws.amazon.com/blogs/security/post-quantum-tls-now-supported-in-aws-kms/
*
* Since this Cipher Preference contains algorithms still being evaluated by NIST, it may stop being supported at
* any time.
* @deprecated This TlsCipherPreference is no longer supported. Use TLS_CIPHER_PREF_PQ_TLSv1_0_2021_05 instead.
*/
@Deprecated
TLS_CIPHER_PREF_KMS_PQ_SIKE_TLSv1_0_2020_02(4),

/**
* This TlsCipherPreference contains Kyber Round 2, BIKE Round 2, SIKE Round 2, BIKE Round 1, and SIKE Round 1 Draft
* Hybrid TLS Ciphers at the top of the preference list.
*
* For more info see:
* - https://tools.ietf.org/html/draft-campagna-tls-bike-sike-hybrid
* - https://aws.amazon.com/blogs/security/post-quantum-tls-now-supported-in-aws-kms/
*
* Since this Cipher Preference contains algorithms still being evaluated by NIST, it may stop being supported at
* any time.
* @deprecated This TlsCipherPreference is no longer supported. Use TLS_CIPHER_PREF_PQ_TLSv1_0_2021_05 instead.
*/
@Deprecated
TLS_CIPHER_PREF_KMS_PQ_TLSv1_0_2020_07(5),

/**
* This TlsCipherPreference supports TLS 1.0 through TLS 1.3, as well as supporting Kyber Round 3, Bike Round 3,
* and SIKE Round 3.
* This TlsCipherPreference supports TLS 1.0 through TLS 1.3, and contains Kyber Round 3 as its highest priority
* PQ algorithm. PQ algorithms in this preference list will be used in hybrid mode, and will be combined with a
* classical ECDHE key exchange.
*
* NIST has announced that Kyber will be first post-quantum key-agreement algorithm that it will standardize.
* However, the NIST standardization process might introduce minor changes that may cause the final Kyber standard
* to differ from the Kyber Round 3 implementation available in this preference list.
*
* Since this TlsCipherPreference contains algorithms that have not yet been officially standardized by NIST, this
* preference list, and any of the PQ algorithms in it, may stop being supported at any time.
*
* For more info see:
* - https://tools.ietf.org/html/draft-campagna-tls-bike-sike-hybrid
* - https://datatracker.ietf.org/doc/html/draft-ietf-tls-hybrid-design
* - https://aws.amazon.com/blogs/security/post-quantum-tls-now-supported-in-aws-kms/
*
* Since this Cipher Preference contains algorithms still being evaluated by NIST, it may stop being supported at
* any time.
* - https://aws.amazon.com/blogs/security/how-to-tune-tls-for-hybrid-post-quantum-cryptography-with-kyber/
* - https://nvlpubs.nist.gov/nistpubs/ir/2022/NIST.IR.8413.pdf
*/
TLS_CIPHER_PREF_PQ_TLSv1_0_2021_05(6);

Expand Down
110 changes: 110 additions & 0 deletions src/main/java/software/amazon/awssdk/crt/io/TlsConnectionOptions.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
/**
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0.
*/
package software.amazon.awssdk.crt.io;

import java.util.List;
import java.util.ArrayList;

import software.amazon.awssdk.crt.CrtResource;
import software.amazon.awssdk.crt.utils.StringUtils;

/**
* Connection-specific TLS options.
*/
public class TlsConnectionOptions extends CrtResource {
private List<String> alpnList = new ArrayList<>();
private String serverName = null;
private int timeoutMs = 0;
private TlsContext tlsContext;

/**
* Initialize the connection-specific TLS options with TLSContext.
* @param tlsContext the TLS configuration contexts in the AWS Common Runtime.
*/
public TlsConnectionOptions(TlsContext tlsContext) {
this.tlsContext = tlsContext;
}

/**
* Note: Once this gets invoked the native resource will not be able to changed.
*/
@Override
public long getNativeHandle() {
if (super.getNativeHandle() == 0) {
acquireNativeHandle(tlsConnectionOptionsNew(
alpnList.size() > 0 ? StringUtils.join(";", alpnList) : null,
serverName,
timeoutMs,
tlsContext.getNativeHandle()));
}
return super.getNativeHandle();
}

/**
* Sets alpn list in the form protocol1;protocol2;.... A maximum of 4
* protocols are supported.
* alpnList is copied. This value is already inherited from TlsContext, but the
* TlsContext is expensive, and should be used across as many connections as
* possible. If you want to set this per connection, set it here.
* @param alpnList Semi-colon delimited list of supported ALPN protocols
* @return this
*/
public TlsConnectionOptions withAlpnList(String alpnList) {
String[] parts = alpnList.split(";");
for (String part : parts) {
this.alpnList.add(part);
}
return this;
}

/**
* Sets server name to use for the SNI extension (supported everywhere), as well
* as x.509 validation. If you don't set this, your x.509 validation will likely
* fail.
* @param serverName The server name to use for the SNI extension
* @return this
*/
public TlsConnectionOptions withServerName(String serverName) {
this.serverName = serverName;
return this;
}
/**
* Set the TLS negotiation timeout
* @param timeoutMs The time out in ms
* @return this
*/
public TlsConnectionOptions withTimeoutMs(int timeoutMs) {
this.timeoutMs = timeoutMs;
return this;
}

/**
* Determines whether a resource releases its dependencies at the same time the
* native handle is released or if it waits.
* Resources that wait are responsible for calling releaseReferences() manually.
*/
@Override
protected boolean canReleaseReferencesImmediately() {
return true;
}

/**
* Cleans up the client bootstrap's associated native handle
*/
@Override
protected void releaseNativeHandle() {
if (!isNull()) {
tlsConnectionOptionsDestroy(getNativeHandle());
}
}

/*******************************************************************************
* native methods
******************************************************************************/
private static native long tlsConnectionOptionsNew(
String alpn, String serverName, int connectTimeoutMs, long tlsContext);

private static native void tlsConnectionOptionsDestroy(long tlsOptions);
}
Loading

0 comments on commit 868747c

Please sign in to comment.