Skip to content

Commit

Permalink
Update to use stream activiation refactor in aws-c-http. (#104)
Browse files Browse the repository at this point in the history
* Update to use stream activiation refactor in aws-c-http.
* Updated default args for allocators to use the configured one. Updated STL allocator to allow PMR style allocators where it makes sense. Updated http to use the new stream activate() apis. Added test for stream that isn't activated.
* Updated to latest http version, updated api contract and documentation to reflect.
* Update to latest version of all crt libs.
  • Loading branch information
JonathanHenson authored Mar 27, 2020
1 parent 7d3c0b4 commit 4f2eb5d
Show file tree
Hide file tree
Showing 25 changed files with 246 additions and 97 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ on:
- '!master'

env:
BUILDER_VERSION: v0.5.7
BUILDER_VERSION: v0.5.9
BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net
PACKAGE_NAME: aws-crt-cpp
LINUX_BASE_IMAGE: ubuntu-16-x64
Expand Down
2 changes: 1 addition & 1 deletion aws-common-runtime/aws-c-common
Submodule aws-c-common updated 42 files
+36 −9 .cbmc-batch/cbmc-batch.sh
+7 −6 .cbmc-batch/check_result.py
+3 −3 .cbmc-batch/include/proof_helpers/aws_byte_cursor_read_common.h
+0 −1 .cbmc-batch/jobs/aws_array_list_back/Makefile
+1 −3 .cbmc-batch/jobs/aws_array_list_back/aws_array_list_back_harness.c
+2 −2 .cbmc-batch/jobs/aws_array_list_back/cbmc-batch.yaml
+0 −1 .cbmc-batch/jobs/aws_array_list_front/Makefile
+1 −3 .cbmc-batch/jobs/aws_array_list_front/aws_array_list_front_harness.c
+2 −2 .cbmc-batch/jobs/aws_array_list_front/cbmc-batch.yaml
+0 −1 .cbmc-batch/jobs/aws_array_list_get_at/Makefile
+1 −3 .cbmc-batch/jobs/aws_array_list_get_at/aws_array_list_get_at_harness.c
+2 −2 .cbmc-batch/jobs/aws_array_list_get_at/cbmc-batch.yaml
+0 −1 .cbmc-batch/jobs/aws_array_list_push_back/Makefile
+2 −4 .cbmc-batch/jobs/aws_array_list_push_back/aws_array_list_push_back_harness.c
+2 −2 .cbmc-batch/jobs/aws_array_list_push_back/cbmc-batch.yaml
+0 −1 .cbmc-batch/jobs/aws_array_list_set_at/Makefile
+2 −3 .cbmc-batch/jobs/aws_array_list_set_at/aws_array_list_set_at_harness.c
+2 −2 .cbmc-batch/jobs/aws_array_list_set_at/cbmc-batch.yaml
+0 −1 .cbmc-batch/jobs/aws_byte_buf_write_from_whole_cursor/Makefile
+0 −1 .cbmc-batch/jobs/aws_byte_buf_write_from_whole_cursor/aws_byte_buf_write_from_whole_cursor_harness.c
+2 −2 .cbmc-batch/jobs/aws_byte_buf_write_from_whole_cursor/cbmc-batch.yaml
+2 −2 .cbmc-batch/jobs/aws_byte_cursor_read_be16/cbmc-batch.yaml
+1 −0 .cbmc-batch/jobs/aws_byte_cursor_read_u8/aws_byte_cursor_read_u8_harness.c
+2 −2 .cbmc-batch/jobs/aws_byte_cursor_read_u8/cbmc-batch.yaml
+109 −0 .github/workflows/ci.yml
+0 −21 .github/workflows/mac-osx-ci.yml
+9 −1 CMakeLists.txt
+4 −1 cmake/aws-c-common-config.cmake
+6 −0 include/aws/common/array_list.h
+11 −0 include/aws/common/array_list.inl
+9 −6 include/aws/common/assert.h
+62 −0 include/aws/common/bigint.h
+1 −0 include/aws/common/error.h
+58 −0 include/aws/common/math.h
+142 −0 include/aws/common/math.inl
+40 −0 include/aws/testing/aws_test_harness.h
+1,055 −3 source/bigint.c
+6 −0 source/byte_buf.c
+3 −0 source/common.c
+18 −0 tests/CMakeLists.txt
+1,448 −0 tests/bigint_test.c
+258 −0 tests/math_test.c
2 changes: 1 addition & 1 deletion aws-common-runtime/aws-c-http
Submodule aws-c-http updated 60 files
+27 −9 .github/workflows/ci.yml
+1 −0 bin/elasticurl/main.c
+2 −5 builder.json
+11 −0 include/aws/http/connection.h
+5 −0 include/aws/http/connection_manager.h
+0 −2 include/aws/http/http.h
+9 −1 include/aws/http/private/connection_impl.h
+2 −0 include/aws/http/private/h1_connection.h
+44 −2 include/aws/http/private/h1_decoder.h
+1 −1 include/aws/http/private/h1_encoder.h
+4 −0 include/aws/http/private/h1_stream.h
+30 −1 include/aws/http/private/h2_connection.h
+60 −14 include/aws/http/private/h2_decoder.h
+127 −239 include/aws/http/private/h2_frames.h
+7 −8 include/aws/http/private/h2_stream.h
+73 −23 include/aws/http/private/hpack.h
+1 −62 include/aws/http/private/http_impl.h
+1 −0 include/aws/http/private/request_response_impl.h
+1 −0 include/aws/http/private/websocket_impl.h
+61 −25 include/aws/http/request_response.h
+11 −0 include/aws/http/server.h
+92 −0 include/aws/http/status_code.h
+46 −32 source/connection.c
+9 −4 source/connection_manager.c
+88 −44 source/h1_connection.c
+8 −8 source/h1_decoder.c
+3 −3 source/h1_encoder.c
+14 −12 source/h1_stream.c
+470 −30 source/h2_connection.c
+458 −623 source/h2_decoder.c
+854 −713 source/h2_frames.c
+57 −13 source/h2_stream.c
+663 −179 source/hpack.c
+68 −66 source/http.c
+2 −0 source/proxy_connection.c
+40 −16 source/request_response.c
+2 −22 source/websocket.c
+8 −6 source/websocket_bootstrap.c
+111 −11 tests/CMakeLists.txt
+261 −186 tests/fuzz/fuzz_h2_decoder_correct.c
+3 −0 tests/fuzz/fuzz_h2_decoder_random.c
+510 −0 tests/h2_test_helper.c
+159 −0 tests/h2_test_helper.h
+2 −1 tests/integration_test_proxy.c
+8 −6 tests/proxy_test_helper.c
+6 −2 tests/test_connection.c
+4 −5 tests/test_connection_monitor.c
+56 −47 tests/test_h1_client.c
+2 −2 tests/test_h1_decoder.c
+5 −8 tests/test_h1_server.c
+93 −4 tests/test_h2_client.c
+2,362 −0 tests/test_h2_decoder.c
+123 −110 tests/test_h2_encoder.c
+120 −132 tests/test_h2_headers.c
+355 −71 tests/test_hpack.c
+8 −6 tests/test_message.c
+5 −2 tests/test_proxy.c
+1 −0 tests/test_tls.c
+13 −0 tests/test_websocket_bootstrap.c
+15 −7 tests/test_websocket_handler.c
22 changes: 15 additions & 7 deletions include/aws/crt/StlAllocator.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,16 @@ namespace Aws
public:
using Base = std::allocator<T>;

StlAllocator() noexcept : Base() {}
StlAllocator(const StlAllocator<T> &a) noexcept : Base(a) {}
StlAllocator() noexcept : Base() { m_allocator = g_allocator; }

template <class U> StlAllocator(const StlAllocator<U> &a) noexcept : Base(a) {}
StlAllocator(Allocator *allocator) noexcept : Base() { m_allocator = allocator; }

StlAllocator(const StlAllocator<T> &a) noexcept : Base(a) { m_allocator = a.m_allocator; }

template <class U> StlAllocator(const StlAllocator<U> &a) noexcept : Base(a)
{
m_allocator = a.m_allocator;
}

~StlAllocator() {}

Expand All @@ -47,15 +53,17 @@ namespace Aws
typename Base::pointer allocate(size_type n, const void *hint = nullptr)
{
(void)hint;
AWS_ASSERT(g_allocator);
return reinterpret_cast<typename Base::pointer>(aws_mem_acquire(g_allocator, n * sizeof(T)));
AWS_ASSERT(m_allocator);
return reinterpret_cast<typename Base::pointer>(aws_mem_acquire(m_allocator, n * sizeof(T)));
}

void deallocate(typename Base::pointer p, size_type)
{
AWS_ASSERT(g_allocator);
aws_mem_release(g_allocator, p);
AWS_ASSERT(m_allocator);
aws_mem_release(m_allocator, p);
}

Allocator *m_allocator;
};
} // namespace Crt
} // namespace Aws
22 changes: 10 additions & 12 deletions include/aws/crt/auth/Credentials.h
Original file line number Diff line number Diff line change
Expand Up @@ -40,12 +40,12 @@ namespace Aws
class AWS_CRT_CPP_API Credentials
{
public:
Credentials(aws_credentials *credentials, Allocator *allocator = DefaultAllocator()) noexcept;
Credentials(aws_credentials *credentials, Allocator *allocator = g_allocator) noexcept;
Credentials(
ByteCursor access_key_id,
ByteCursor secret_access_key,
ByteCursor session_token,
Allocator *allocator = DefaultAllocator()) noexcept;
Allocator *allocator = g_allocator) noexcept;

~Credentials();

Expand Down Expand Up @@ -241,9 +241,7 @@ namespace Aws
class AWS_CRT_CPP_API CredentialsProvider : public ICredentialsProvider
{
public:
CredentialsProvider(
aws_credentials_provider *provider,
Allocator *allocator = DefaultAllocator()) noexcept;
CredentialsProvider(aws_credentials_provider *provider, Allocator *allocator = g_allocator) noexcept;

virtual ~CredentialsProvider();

Expand Down Expand Up @@ -278,43 +276,43 @@ namespace Aws
*/
static std::shared_ptr<ICredentialsProvider> CreateCredentialsProviderStatic(
const CredentialsProviderStaticConfig &config,
Allocator *allocator = DefaultAllocator());
Allocator *allocator = g_allocator);

/**
* Creates a provider that returns credentials sourced from environment variables
*/
static std::shared_ptr<ICredentialsProvider> CreateCredentialsProviderEnvironment(
Allocator *allocator = DefaultAllocator());
Allocator *allocator = g_allocator);

/**
* Creates a provider that returns credentials sourced from config files
*/
static std::shared_ptr<ICredentialsProvider> CreateCredentialsProviderProfile(
const CredentialsProviderProfileConfig &config,
Allocator *allocator = DefaultAllocator());
Allocator *allocator = g_allocator);

/**
* Creates a provider that returns credentials sourced from Ec2 instance metadata service
*/
static std::shared_ptr<ICredentialsProvider> CreateCredentialsProviderImds(
const CredentialsProviderImdsConfig &config,
Allocator *allocator = DefaultAllocator());
Allocator *allocator = g_allocator);

/**
* Creates a provider that sources credentials by querying a series of providers and
* returning the first valid credential set encountered
*/
static std::shared_ptr<ICredentialsProvider> CreateCredentialsProviderChain(
const CredentialsProviderChainConfig &config,
Allocator *allocator = DefaultAllocator());
Allocator *allocator = g_allocator);

/*
* Creates a provider that puts a simple time-based cache in front of its queries
* to a subordinate provider.
*/
static std::shared_ptr<ICredentialsProvider> CreateCredentialsProviderCached(
const CredentialsProviderCachedConfig &config,
Allocator *allocator = DefaultAllocator());
Allocator *allocator = g_allocator);

/**
* Creates the SDK-standard default credentials provider which is a cache-fronted chain of:
Expand All @@ -324,7 +322,7 @@ namespace Aws
*/
static std::shared_ptr<ICredentialsProvider> CreateCredentialsProviderChainDefault(
const CredentialsProviderChainDefaultConfig &config,
Allocator *allocator = DefaultAllocator());
Allocator *allocator = g_allocator);

private:
static void s_onCredentialsResolved(aws_credentials *credentials, void *user_data);
Expand Down
4 changes: 2 additions & 2 deletions include/aws/crt/auth/Sigv4Signing.h
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ namespace Aws
class AWS_CRT_CPP_API AwsSigningConfig : public ISigningConfig
{
public:
AwsSigningConfig(Allocator *allocator = DefaultAllocator());
AwsSigningConfig(Allocator *allocator = g_allocator);
virtual ~AwsSigningConfig();

virtual SigningConfigType GetType() const noexcept override { return SigningConfigType::Aws; }
Expand Down Expand Up @@ -178,7 +178,7 @@ namespace Aws
class AWS_CRT_CPP_API Sigv4HttpRequestSigner : public IHttpRequestSigner
{
public:
Sigv4HttpRequestSigner(Allocator *allocator = DefaultAllocator());
Sigv4HttpRequestSigner(Allocator *allocator = g_allocator);
virtual ~Sigv4HttpRequestSigner() = default;

bool IsValid() const override { return true; }
Expand Down
4 changes: 2 additions & 2 deletions include/aws/crt/crypto/Hash.h
Original file line number Diff line number Diff line change
Expand Up @@ -94,12 +94,12 @@ namespace Aws
/**
* Creates an instance of a Streaming SHA256 Hash.
*/
static Hash CreateSHA256(Allocator *allocator = DefaultAllocator()) noexcept;
static Hash CreateSHA256(Allocator *allocator = g_allocator) noexcept;

/**
* Creates an instance of a Streaming MD5 Hash.
*/
static Hash CreateMD5(Allocator *allocator = DefaultAllocator()) noexcept;
static Hash CreateMD5(Allocator *allocator = g_allocator) noexcept;

/**
* Updates the running hash object with data in toHash. Returns true on success. Call
Expand Down
30 changes: 28 additions & 2 deletions include/aws/crt/http/HttpConnection.h
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ namespace Aws
* Represents a single http message exchange (request/response) or in H2, it can also represent
* a PUSH_PROMISE followed by the accompanying Response.
*/
class AWS_CRT_CPP_API HttpStream
class AWS_CRT_CPP_API HttpStream : public std::enable_shared_from_this<HttpStream>
{
public:
virtual ~HttpStream();
Expand Down Expand Up @@ -184,10 +184,17 @@ namespace Aws
friend class HttpClientConnection;
};

struct ClientStreamCallbackData
{
ClientStreamCallbackData() : allocator(nullptr), stream(nullptr) {}
Allocator *allocator;
std::shared_ptr<HttpStream> stream;
};

class AWS_CRT_CPP_API HttpClientStream final : public HttpStream
{
public:
~HttpClientStream() = default;
~HttpClientStream();
HttpClientStream(const HttpClientStream &) = delete;
HttpClientStream(HttpClientStream &&) = delete;
HttpClientStream &operator=(const HttpClientStream &) = delete;
Expand All @@ -199,9 +206,17 @@ namespace Aws
*/
virtual int GetResponseStatusCode() const noexcept override;

/**
* Activates the request's outgoing stream processing.
*
* Returns true on success, false otherwise.
*/
bool Activate() noexcept;

private:
HttpClientStream(const std::shared_ptr<HttpClientConnection> &connection) noexcept;

ClientStreamCallbackData m_callbackData;
friend class HttpClientConnection;
};

Expand Down Expand Up @@ -333,6 +348,15 @@ namespace Aws
* Optional.
*/
Optional<HttpClientConnectionProxyOptions> ProxyOptions;

/**
* If set to true, then the TCP read back pressure mechanism will be enabled. You should
* only use this if you're allowing http response body data to escape the callbacks. E.g. you're
* putting the data into a queue for another thread to process and need to make sure the memory
* usage is bounded. If this is enabled, you must call HttpStream::UpdateWindow() for every
* byte read from the OnIncomingBody callback.
*/
bool ManualWindowManagement;
};

/**
Expand All @@ -356,6 +380,8 @@ namespace Aws
* not be freed until the stream is completed.
*
* Returns an instance of HttpStream upon success and nullptr on failure.
*
* You must call HttpClientStream::Activate() to begin outgoing processing of the stream.
*/
std::shared_ptr<HttpClientStream> NewClientStream(const HttpRequestOptions &requestOptions) noexcept;

Expand Down
4 changes: 2 additions & 2 deletions include/aws/crt/http/HttpConnectionManager.h
Original file line number Diff line number Diff line change
Expand Up @@ -99,12 +99,12 @@ namespace Aws
*/
static std::shared_ptr<HttpClientConnectionManager> NewClientConnectionManager(
const HttpClientConnectionManagerOptions &connectionManagerOptions,
Allocator *allocator = DefaultAllocator()) noexcept;
Allocator *allocator = g_allocator) noexcept;

private:
HttpClientConnectionManager(
const HttpClientConnectionManagerOptions &options,
Allocator *allocator = DefaultAllocator()) noexcept;
Allocator *allocator = g_allocator) noexcept;

Allocator *m_allocator;

Expand Down
4 changes: 2 additions & 2 deletions include/aws/crt/http/HttpRequestResponse.h
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ namespace Aws
friend class Mqtt::MqttConnection;

public:
HttpRequest(Allocator *allocator = DefaultAllocator());
HttpRequest(Allocator *allocator = g_allocator);

/**
* Gets the value of the Http method associated with this request
Expand Down Expand Up @@ -117,7 +117,7 @@ namespace Aws
class AWS_CRT_CPP_API HttpResponse : public HttpMessage
{
public:
HttpResponse(Allocator *allocator = DefaultAllocator());
HttpResponse(Allocator *allocator = g_allocator);

/**
* Gets the integral Http response code associated with this response
Expand Down
2 changes: 1 addition & 1 deletion include/aws/crt/io/Bootstrap.h
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ namespace Aws
ClientBootstrap(
EventLoopGroup &elGroup,
HostResolver &resolver,
Allocator *allocator = DefaultAllocator()) noexcept;
Allocator *allocator = g_allocator) noexcept;
~ClientBootstrap();
ClientBootstrap(const ClientBootstrap &) = delete;
ClientBootstrap &operator=(const ClientBootstrap &) = delete;
Expand Down
4 changes: 2 additions & 2 deletions include/aws/crt/io/EventLoopGroup.h
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ namespace Aws
class AWS_CRT_CPP_API EventLoopGroup final
{
public:
EventLoopGroup(Allocator *allocator = DefaultAllocator()) noexcept;
EventLoopGroup(uint16_t threadCount, Allocator *allocator = DefaultAllocator()) noexcept;
EventLoopGroup(Allocator *allocator = g_allocator) noexcept;
EventLoopGroup(uint16_t threadCount, Allocator *allocator = g_allocator) noexcept;
~EventLoopGroup();
EventLoopGroup(const EventLoopGroup &) = delete;
EventLoopGroup(EventLoopGroup &&) noexcept;
Expand Down
2 changes: 1 addition & 1 deletion include/aws/crt/io/HostResolver.h
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ namespace Aws
EventLoopGroup &elGroup,
size_t maxHosts,
size_t maxTTL,
Allocator *allocator = DefaultAllocator()) noexcept;
Allocator *allocator = g_allocator) noexcept;
~DefaultHostResolver();
DefaultHostResolver(const DefaultHostResolver &) = delete;
DefaultHostResolver &operator=(const DefaultHostResolver &) = delete;
Expand Down
4 changes: 2 additions & 2 deletions include/aws/crt/io/Stream.h
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ namespace Aws
Allocator *m_allocator;
aws_input_stream m_underlying_stream;

InputStream(Aws::Crt::Allocator *allocator = DefaultAllocator());
InputStream(Aws::Crt::Allocator *allocator = g_allocator);

/***
* Read up-to buffer::capacity - buffer::len into buffer::buffer
Expand Down Expand Up @@ -109,7 +109,7 @@ namespace Aws
public:
StdIOStreamInputStream(
std::shared_ptr<Aws::Crt::Io::IStream> stream,
Aws::Crt::Allocator *allocator = DefaultAllocator()) noexcept;
Aws::Crt::Allocator *allocator = g_allocator) noexcept;

bool IsValid() const noexcept override;

Expand Down
13 changes: 5 additions & 8 deletions include/aws/crt/io/TlsOptions.h
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ namespace Aws
* Initializes TlsContextOptions with secure by default options, with
* no client certificates.
*/
static TlsContextOptions InitDefaultClient(Allocator *allocator = DefaultAllocator()) noexcept;
static TlsContextOptions InitDefaultClient(Allocator *allocator = g_allocator) noexcept;
/**
* Initializes TlsContextOptions with secure by default options, with
* client certificate and private key. These are paths to a file on disk. These files
Expand All @@ -59,7 +59,7 @@ namespace Aws
static TlsContextOptions InitClientWithMtls(
const char *cert_path,
const char *pkey_path,
Allocator *allocator = DefaultAllocator()) noexcept;
Allocator *allocator = g_allocator) noexcept;

/**
* Initializes TlsContextOptions with secure by default options, with
Expand All @@ -69,7 +69,7 @@ namespace Aws
static TlsContextOptions InitClientWithMtls(
const ByteCursor &cert,
const ByteCursor &pkey,
Allocator *allocator = DefaultAllocator()) noexcept;
Allocator *allocator = g_allocator) noexcept;

#ifdef __APPLE__
/**
Expand All @@ -81,7 +81,7 @@ namespace Aws
static TlsContextOptions InitClientWithMtlsPkcs12(
const char *pkcs12_path,
const char *pkcs12_pwd,
Allocator *allocator = DefaultAllocator()) noexcept;
Allocator *allocator = g_allocator) noexcept;
#endif

/**
Expand Down Expand Up @@ -172,10 +172,7 @@ namespace Aws
{
public:
TlsContext() noexcept;
TlsContext(
TlsContextOptions &options,
TlsMode mode,
Allocator *allocator = DefaultAllocator()) noexcept;
TlsContext(TlsContextOptions &options, TlsMode mode, Allocator *allocator = g_allocator) noexcept;
~TlsContext() = default;
TlsContext(const TlsContext &) noexcept = default;
TlsContext &operator=(const TlsContext &) noexcept = default;
Expand Down
4 changes: 2 additions & 2 deletions include/aws/crt/io/Uri.h
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,12 @@ namespace Aws
* Parses `cursor` as a URI. Upon failure the bool() operator will return false and LastError()
* will contain the errorCode.
*/
Uri(const ByteCursor &cursor, Allocator *allocator = DefaultAllocator()) noexcept;
Uri(const ByteCursor &cursor, Allocator *allocator = g_allocator) noexcept;
/**
* builds a URI from `builderOptions`. Upon failure the bool() operator will return false and
* LastError() will contain the errorCode.
*/
Uri(aws_uri_builder_options &builderOptions, Allocator *allocator = DefaultAllocator()) noexcept;
Uri(aws_uri_builder_options &builderOptions, Allocator *allocator = g_allocator) noexcept;
Uri(const Uri &);
Uri &operator=(const Uri &);
Uri(Uri &&uri) noexcept;
Expand Down
Loading

0 comments on commit 4f2eb5d

Please sign in to comment.