diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1913a278a1..b7d99c776f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -48,6 +48,7 @@ add_subdirectory(ccontrol) add_subdirectory(css) add_subdirectory(czar) add_subdirectory(global) +add_subdirectory(http) add_subdirectory(memman) add_subdirectory(mimic) add_subdirectory(mysql) diff --git a/src/replica/HttpAsyncReq.cc b/src/http/AsyncReq.cc similarity index 68% rename from src/replica/HttpAsyncReq.cc rename to src/http/AsyncReq.cc index 243143f0c6..a4aba96b18 100644 --- a/src/replica/HttpAsyncReq.cc +++ b/src/http/AsyncReq.cc @@ -20,7 +20,7 @@ */ // Class header -#include "replica/HttpAsyncReq.h" +#include "http/AsyncReq.h" // Standard headers #include @@ -31,7 +31,7 @@ #include "boost/date_time/posix_time/posix_time.hpp" // Qserv headers -#include "replica/HttpExceptions.h" +#include "http/Exceptions.h" // LSST headers #include "lsst/log/Log.h" @@ -42,7 +42,7 @@ namespace http = boost::beast::http; namespace { -LOG_LOGGER _log = LOG_GET("lsst.qserv.replica.HttpAsyncReq"); +LOG_LOGGER _log = LOG_GET("lsst.qserv.http.AsyncReq"); http::verb method2verb(string const& method) { if (method == "GET") @@ -53,13 +53,13 @@ http::verb method2verb(string const& method) { return http::verb::put; else if (method == "DELETE") return http::verb::delete_; - throw invalid_argument("HttpAsyncReq::" + string(__func__) + " invalid method '" + method + "'."); + throw invalid_argument("AsyncReq::" + string(__func__) + " invalid method '" + method + "'."); } } // namespace -namespace lsst::qserv::replica { +namespace lsst::qserv::http { -string HttpAsyncReq::state2str(State state) { +string AsyncReq::state2str(State state) { switch (state) { case State::CREATED: return "CREATED"; @@ -76,22 +76,22 @@ string HttpAsyncReq::state2str(State state) { case State::EXPIRED: return "EXPIRED"; } - throw invalid_argument("HttpAsyncReq::" + string(__func__) + + throw invalid_argument("AsyncReq::" + string(__func__) + " unknown state: " + to_string(static_cast(state)) + "."); } -shared_ptr HttpAsyncReq::create(asio::io_service& io_service, CallbackType const& onFinish, - string const& method, string const& url, string const& data, - std::unordered_map const& headers, - size_t maxResponseBodySize, unsigned int expirationIvalSec) { - return shared_ptr(new HttpAsyncReq(io_service, onFinish, method, url, data, headers, - maxResponseBodySize, expirationIvalSec)); +shared_ptr AsyncReq::create(asio::io_service& io_service, CallbackType const& onFinish, + string const& method, string const& url, string const& data, + std::unordered_map const& headers, + size_t maxResponseBodySize, unsigned int expirationIvalSec) { + return shared_ptr(new AsyncReq(io_service, onFinish, method, url, data, headers, + maxResponseBodySize, expirationIvalSec)); } -HttpAsyncReq::HttpAsyncReq(asio::io_service& io_service, CallbackType const& onFinish, string const& method, - string const& url, string const& data, - std::unordered_map const& headers, - size_t maxResponseBodySize, unsigned int expirationIvalSec) +AsyncReq::AsyncReq(asio::io_service& io_service, CallbackType const& onFinish, string const& method, + string const& url, string const& data, + std::unordered_map const& headers, size_t maxResponseBodySize, + unsigned int expirationIvalSec) : _io_service(io_service), _resolver(io_service), _socket(io_service), @@ -104,7 +104,7 @@ HttpAsyncReq::HttpAsyncReq(asio::io_service& io_service, CallbackType const& onF _expirationIvalSec(expirationIvalSec), _expirationTimer(io_service), _timer(io_service) { - string const context = "HttpAsyncReq::" + string(__func__) + " "; + string const context = "AsyncReq::" + string(__func__) + " "; if (_url.scheme() != Url::Scheme::HTTP) { throw invalid_argument(context + "this implementation only supports urls based on the HTTP scheme."); } @@ -123,14 +123,14 @@ HttpAsyncReq::HttpAsyncReq(asio::io_service& io_service, CallbackType const& onF } } -HttpAsyncReq::~HttpAsyncReq() { +AsyncReq::~AsyncReq() { _expirationTimer.cancel(); _timer.cancel(); boost::system::error_code ec; _socket.shutdown(asio::ip::tcp::socket::shutdown_both, ec); } -string HttpAsyncReq::version() const { +string AsyncReq::version() const { switch (_req.version()) { case 11: return "HTTP/1.1"; @@ -139,9 +139,9 @@ string HttpAsyncReq::version() const { } } -void HttpAsyncReq::start() { - string const context = "HttpAsyncReq::" + string(__func__) + " "; - replica::Lock lock(_mtx, context); +void AsyncReq::start() { + string const context = "AsyncReq::" + string(__func__) + " "; + std::lock_guard const lock(_mtx); _assertState(lock, context, {State::CREATED}); try { _resolve(lock); @@ -157,9 +157,9 @@ void HttpAsyncReq::start() { } } -bool HttpAsyncReq::cancel() { - string const context = "HttpAsyncReq::" + string(__func__) + " "; - replica::Lock lock(_mtx, context); +bool AsyncReq::cancel() { + string const context = "AsyncReq::" + string(__func__) + " "; + std::lock_guard const lock(_mtx); switch (_state) { case State::CREATED: case State::IN_PROGRESS: @@ -170,62 +170,62 @@ bool HttpAsyncReq::cancel() { } } -string HttpAsyncReq::errorMessage() const { - string const context = "HttpAsyncReq::" + string(__func__) + " "; - replica::Lock lock(_mtx, context); +string AsyncReq::errorMessage() const { + string const context = "AsyncReq::" + string(__func__) + " "; + std::lock_guard const lock(_mtx); return _error; } -int HttpAsyncReq::responseCode() const { - string const context = "HttpAsyncReq::" + string(__func__) + " "; - replica::Lock lock(_mtx, context); +int AsyncReq::responseCode() const { + string const context = "AsyncReq::" + string(__func__) + " "; + std::lock_guard const lock(_mtx); _assertState(lock, context, {State::FINISHED, State::BODY_LIMIT_ERROR}); auto const& header = _res.get().base(); return header.result_int(); } -unordered_map const& HttpAsyncReq::responseHeader() const { - string const context = "HttpAsyncReq::" + string(__func__) + " "; - replica::Lock lock(_mtx, context); +unordered_map const& AsyncReq::responseHeader() const { + string const context = "AsyncReq::" + string(__func__) + " "; + std::lock_guard const lock(_mtx); _assertState(lock, context, {State::FINISHED, State::BODY_LIMIT_ERROR}); return _responseHeader; } -string const& HttpAsyncReq::responseBody() const { - string const context = "HttpAsyncReq::" + string(__func__) + " "; - replica::Lock lock(_mtx, context); +string const& AsyncReq::responseBody() const { + string const context = "AsyncReq::" + string(__func__) + " "; + std::lock_guard const lock(_mtx); _assertState(lock, context, {State::FINISHED}); return _res.get().body(); } -size_t HttpAsyncReq::responseBodySize() const { - string const context = "HttpAsyncReq::" + string(__func__) + " "; - replica::Lock lock(_mtx, context); +size_t AsyncReq::responseBodySize() const { + string const context = "AsyncReq::" + string(__func__) + " "; + std::lock_guard const lock(_mtx); _assertState(lock, context, {State::FINISHED}); return _res.get().body().size(); } -void HttpAsyncReq::_restart(replica::Lock const& lock) { +void AsyncReq::_restart(std::lock_guard const& lock) { _timer.cancel(); _timer.expires_from_now(boost::posix_time::seconds(_timerIvalSec)); _timer.async_wait( [self = shared_from_this()](boost::system::error_code const& ec) { self->_restarted(ec); }); } -void HttpAsyncReq::_restarted(boost::system::error_code const& ec) { - string const context = "HttpAsyncReq::" + string(__func__) + " "; +void AsyncReq::_restarted(boost::system::error_code const& ec) { + string const context = "AsyncReq::" + string(__func__) + " "; // Ignore this event if the timer was aborted if (ec == boost::asio::error::operation_aborted) return; if (State::IN_PROGRESS != _state) return; - replica::Lock lock(_mtx, context); + std::lock_guard const lock(_mtx); if (State::IN_PROGRESS != _state) return; _resolve(lock); } -void HttpAsyncReq::_resolve(replica::Lock const& lock) { +void AsyncReq::_resolve(std::lock_guard const& lock) { _resolver.async_resolve( _url.host(), to_string(_url.port() == 0 ? 80 : _url.port()), [self = shared_from_this()](boost::system::error_code const& ec, @@ -234,12 +234,12 @@ void HttpAsyncReq::_resolve(replica::Lock const& lock) { }); } -void HttpAsyncReq::_resolved(boost::system::error_code const& ec, - asio::ip::tcp::resolver::results_type const& results) { - string const context = "HttpAsyncReq::" + string(__func__) + " "; +void AsyncReq::_resolved(boost::system::error_code const& ec, + asio::ip::tcp::resolver::results_type const& results) { + string const context = "AsyncReq::" + string(__func__) + " "; if (State::IN_PROGRESS != _state) return; - replica::Lock lock(_mtx, context); + std::lock_guard const lock(_mtx); if (State::IN_PROGRESS != _state) return; if (ec.value() != 0) { @@ -253,11 +253,11 @@ void HttpAsyncReq::_resolved(boost::system::error_code const& ec, const asio::ip::tcp::endpoint& endpoint) { self->_connected(ec); }); } -void HttpAsyncReq::_connected(boost::system::error_code const& ec) { - string const context = "HttpAsyncReq::" + string(__func__) + " "; +void AsyncReq::_connected(boost::system::error_code const& ec) { + string const context = "AsyncReq::" + string(__func__) + " "; if (State::IN_PROGRESS != _state) return; - replica::Lock lock(_mtx, context); + std::lock_guard const lock(_mtx); if (State::IN_PROGRESS != _state) return; if (ec.value() != 0) { @@ -271,11 +271,11 @@ void HttpAsyncReq::_connected(boost::system::error_code const& ec) { }); } -void HttpAsyncReq::_sent(boost::system::error_code const& ec, size_t bytesSent) { - string const context = "HttpAsyncReq::" + string(__func__) + " "; +void AsyncReq::_sent(boost::system::error_code const& ec, size_t bytesSent) { + string const context = "AsyncReq::" + string(__func__) + " "; if (State::IN_PROGRESS != _state) return; - replica::Lock lock(_mtx, context); + std::lock_guard const lock(_mtx); if (State::IN_PROGRESS != _state) return; if (ec.value() != 0) { @@ -292,11 +292,11 @@ void HttpAsyncReq::_sent(boost::system::error_code const& ec, size_t bytesSent) }); } -void HttpAsyncReq::_received(boost::system::error_code const& ec, size_t bytesReceived) { - string const context = "HttpAsyncReq::" + string(__func__) + " "; +void AsyncReq::_received(boost::system::error_code const& ec, size_t bytesReceived) { + string const context = "AsyncReq::" + string(__func__) + " "; if (_state != State::IN_PROGRESS) return; - replica::Lock lock(_mtx, context); + std::lock_guard const lock(_mtx); if (_state != State::IN_PROGRESS) return; State newState = State::FINISHED; @@ -317,27 +317,27 @@ void HttpAsyncReq::_received(boost::system::error_code const& ec, size_t bytesRe _finish(lock, newState); } -void HttpAsyncReq::_extractCacheHeader(replica::Lock const& lock) { +void AsyncReq::_extractCacheHeader(std::lock_guard const& lock) { auto const& header = _res.get().base(); for (auto itr = header.cbegin(); itr != header.cend(); ++itr) { _responseHeader.insert(pair(itr->name_string(), itr->value())); } } -void HttpAsyncReq::_expired(boost::system::error_code const& ec) { - string const context = "HttpAsyncReq::" + string(__func__) + " "; +void AsyncReq::_expired(boost::system::error_code const& ec) { + string const context = "AsyncReq::" + string(__func__) + " "; // Ignore this event if the timer was aborted if (ec == boost::asio::error::operation_aborted) return; if (_state != State::IN_PROGRESS) return; - replica::Lock lock(_mtx, context); + std::lock_guard const lock(_mtx); if (_state != State::IN_PROGRESS) return; _finish(lock, State::EXPIRED); } -void HttpAsyncReq::_finish(replica::Lock const& lock, State finalState, string const& error) { +void AsyncReq::_finish(std::lock_guard const& lock, State finalState, string const& error) { _state = finalState; _error = error; @@ -360,8 +360,8 @@ void HttpAsyncReq::_finish(replica::Lock const& lock, State finalState, string c } } -void HttpAsyncReq::_assertState(replica::Lock const& lock, string const& context, - initializer_list const& desiredStates) const { +void AsyncReq::_assertState(std::lock_guard const& lock, string const& context, + initializer_list const& desiredStates) const { if (find(desiredStates.begin(), desiredStates.end(), _state) == desiredStates.end()) { string states; for (auto&& state : desiredStates) { @@ -373,11 +373,11 @@ void HttpAsyncReq::_assertState(replica::Lock const& lock, string const& context } } -void HttpAsyncReq::_logError(string const& prefix, boost::system::error_code const& ec) const { +void AsyncReq::_logError(string const& prefix, boost::system::error_code const& ec) const { LOGS(_log, LOG_LVL_WARN, prefix << " method: " << _method << " url: " << _url.url() << " host: " << _url.host() << " port: " << _url.port() << " target: " << _url.target() << " ec: " << ec.value() << " [" << ec.message() << "]"); } -} // namespace lsst::qserv::replica +} // namespace lsst::qserv::http diff --git a/src/replica/HttpAsyncReq.h b/src/http/AsyncReq.h similarity index 82% rename from src/replica/HttpAsyncReq.h rename to src/http/AsyncReq.h index 884c501d3b..6b64c15805 100644 --- a/src/replica/HttpAsyncReq.h +++ b/src/http/AsyncReq.h @@ -18,13 +18,14 @@ * the GNU General Public License along with this program. If not, * see . */ -#ifndef LSST_QSERV_HTTPASYNCREQ_H -#define LSST_QSERV_HTTPASYNCREQ_H +#ifndef LSST_QSERV_HTTP_ASYNCREQ_H +#define LSST_QSERV_HTTP_ASYNCREQ_H // System headers #include #include #include +#include #include #include #include @@ -34,14 +35,13 @@ #include "boost/beast.hpp" // Qserv headers -#include "replica/Url.h" -#include "replica/Mutex.h" +#include "http/Url.h" // This header declarations -namespace lsst::qserv::replica { +namespace lsst::qserv::http { /** - * @brief Class HttpAsyncReq represents a simple asynchronous interface for + * @brief Class AsyncReq represents a simple asynchronous interface for * communicating over the HTTP protocol. * * The implementation of the class invokes a user-supplied callback (lambda) function @@ -51,11 +51,11 @@ namespace lsst::qserv::replica { * to the standard output stream: * @code * boost::asio::io_service io_service; - * std::shared_ptr const reader = - * HttpAsyncReq::create( + * std::shared_ptr const reader = + * http::AsyncReq::create( * [](auto const& reader) { - * if (reader->state() != HttpAsyncReq::State::FINISHED) { - * std::cerr << "request failed, state: " << HttpAsyncReq::state2str(reader->state()) + * if (reader->state() != http::AsyncReq::State::FINISHED) { + * std::cerr << "request failed, state: " << AsyncReq::state2str(reader->state()) * << ", error: " << reader->errorMessage() << std::endl; * return; * } @@ -80,10 +80,10 @@ namespace lsst::qserv::replica { * @note The implementation will open and close a new connection for each request. * @note The implementation doesn't support TLS/SSL-based HTTPS protocol. */ -class HttpAsyncReq : public std::enable_shared_from_this { +class AsyncReq : public std::enable_shared_from_this { public: /// The function type for notifications on the completion of the operation. - typedef std::function const&)> CallbackType; + typedef std::function const&)> CallbackType; enum class State : int { CREATED = 0, ///< The object was created and no request was initiated. @@ -104,9 +104,9 @@ class HttpAsyncReq : public std::enable_shared_from_this { /// @throw std::invalid_argument For unknown values of the input parameter. static std::string state2str(State state); - HttpAsyncReq() = delete; - HttpAsyncReq(HttpAsyncReq const&) = delete; - HttpAsyncReq& operator=(HttpAsyncReq const&) = delete; + AsyncReq() = delete; + AsyncReq(AsyncReq const&) = delete; + AsyncReq& operator=(AsyncReq const&) = delete; /** * @brief Static factory for creating objects of the class. @@ -136,21 +136,20 @@ class HttpAsyncReq : public std::enable_shared_from_this { * @throw std::invalid_argument If empty or invalid values of the input parameters * were provided. */ - static std::shared_ptr create(boost::asio::io_service& io_service, - CallbackType const& onFinish, std::string const& method, - std::string const& url, - std::string const& data = std::string(), - std::unordered_map const& headers = - std::unordered_map(), - size_t maxResponseBodySize = 0, - unsigned int expirationIvalSec = 0); + static std::shared_ptr create(boost::asio::io_service& io_service, CallbackType const& onFinish, + std::string const& method, std::string const& url, + std::string const& data = std::string(), + std::unordered_map const& headers = + std::unordered_map(), + size_t maxResponseBodySize = 0, + unsigned int expirationIvalSec = 0); /// Non-trivial destructor is needed to free up allocated resources. - virtual ~HttpAsyncReq(); + virtual ~AsyncReq(); std::string version() const; std::string const& method() const { return _method; } - Url const& url() const { return _url; } + http::Url const& url() const { return _url; } /// @return The current state of the request. State state() const { return _state; } @@ -198,11 +197,11 @@ class HttpAsyncReq : public std::enable_shared_from_this { size_t responseBodySize() const; private: - /// @see HttpAsyncReq::create() - HttpAsyncReq(boost::asio::io_service& io_service, CallbackType const& onFinish, std::string const& method, - std::string const& url, std::string const& data, - std::unordered_map const& headers, size_t maxResponseBodySize, - unsigned int expirationIvalSec); + /// @see AsyncReq::create() + AsyncReq(boost::asio::io_service& io_service, CallbackType const& onFinish, std::string const& method, + std::string const& url, std::string const& data, + std::unordered_map const& headers, size_t maxResponseBodySize, + unsigned int expirationIvalSec); /** * @brief Verify the desired state against the current one. @@ -212,14 +211,14 @@ class HttpAsyncReq : public std::enable_shared_from_this { * @param desiredStates The desired states (may be more than one) to be verified. * @throw std::logic_error If the current state didn't match the desired one. */ - void _assertState(replica::Lock const& lock, std::string const& context, + void _assertState(std::lock_guard const& lock, std::string const& context, std::initializer_list const& desiredStates) const; // Async operations initiators and handlers. - void _restart(replica::Lock const& lock); + void _restart(std::lock_guard const& lock); void _restarted(boost::system::error_code const& ec); - void _resolve(replica::Lock const& lock); + void _resolve(std::lock_guard const& lock); void _resolved(boost::system::error_code const& ec, boost::asio::ip::tcp::resolver::results_type const& results); void _connected(boost::system::error_code const& ec); @@ -228,7 +227,7 @@ class HttpAsyncReq : public std::enable_shared_from_this { void _expired(boost::system::error_code const& ec); /// Extract the header from the response message and cache it. - void _extractCacheHeader(replica::Lock const& lock); + void _extractCacheHeader(std::lock_guard const& lock); /// Log a error along with the request's parameters in the specified context void _logError(std::string const& prefix, boost::system::error_code const& ec) const; @@ -241,7 +240,8 @@ class HttpAsyncReq : public std::enable_shared_from_this { * @param finalState The final state to be set. * @param error (Optional) The error message to be set. */ - void _finish(replica::Lock const& lock, State finalState, std::string const& error = std::string()); + void _finish(std::lock_guard const& lock, State finalState, + std::string const& error = std::string()); // Data members. @@ -250,7 +250,7 @@ class HttpAsyncReq : public std::enable_shared_from_this { boost::asio::ip::tcp::socket _socket; CallbackType _onFinish; std::string const _method; - Url const _url; + http::Url const _url; std::string const _data; std::unordered_map const _headers; size_t const _maxResponseBodySize; @@ -292,9 +292,9 @@ class HttpAsyncReq : public std::enable_shared_from_this { /// The mutex for enforcing thread safety of the class public API /// and internal operations. - mutable replica::Mutex _mtx; + mutable std::mutex _mtx; }; -} // namespace lsst::qserv::replica +} // namespace lsst::qserv::http -#endif // LSST_QSERV_HTTPASYNCREQ_H +#endif // LSST_QSERV_HTTP_ASYNCREQ_H diff --git a/src/http/CMakeLists.txt b/src/http/CMakeLists.txt new file mode 100644 index 0000000000..64cbabee15 --- /dev/null +++ b/src/http/CMakeLists.txt @@ -0,0 +1,37 @@ +add_library(http SHARED) + +target_sources(http PRIVATE + AsyncReq.cc + Client.cc + Exceptions.cc + Url.cc +) + +target_link_libraries(http PUBLIC + curl + log + qhttp + util + Boost::filesystem + Boost::regex + Boost::system +) + +install(TARGETS http) + +function(http_tests) + foreach(TEST in items ${argv}) + add_executable(${TEST} ${TEST}.cc) + target_link_libraries(${TEST} PUBLIC + http + Boost::unit_test_framework + Threads::Threads + ) + add_test(name ${TEST} command ${TEST}) + endforeach() +endfunction() + +http_tests( + testAsyncReq + testUrl +) diff --git a/src/replica/HttpClient.cc b/src/http/Client.cc similarity index 77% rename from src/replica/HttpClient.cc rename to src/http/Client.cc index d1b1a599e4..7622770f2e 100644 --- a/src/replica/HttpClient.cc +++ b/src/http/Client.cc @@ -20,10 +20,10 @@ */ // Class header -#include "replica/HttpClient.h" +#include "http/Client.h" // Qserv headers -#include "replica/HttpExceptions.h" +#include "http/Exceptions.h" // Standard headers #include @@ -34,55 +34,55 @@ using namespace std; using json = nlohmann::json; -namespace lsst::qserv::replica { +namespace lsst::qserv::http { -string const HttpClientConfig::category = "worker-http-file-reader"; +string const ClientConfig::category = "worker-http-file-reader"; -string const HttpClientConfig::sslVerifyHostKey = "SSL_VERIFYHOST"; -string const HttpClientConfig::sslVerifyPeerKey = "SSL_VERIFYPEER"; -string const HttpClientConfig::caPathKey = "CAPATH"; -string const HttpClientConfig::caInfoKey = "CAINFO"; -string const HttpClientConfig::caInfoValKey = "CAINFO_VAL"; +string const ClientConfig::sslVerifyHostKey = "SSL_VERIFYHOST"; +string const ClientConfig::sslVerifyPeerKey = "SSL_VERIFYPEER"; +string const ClientConfig::caPathKey = "CAPATH"; +string const ClientConfig::caInfoKey = "CAINFO"; +string const ClientConfig::caInfoValKey = "CAINFO_VAL"; -string const HttpClientConfig::proxySslVerifyHostKey = "PROXY_SSL_VERIFYHOST"; -string const HttpClientConfig::proxySslVerifyPeerKey = "PROXY_SSL_VERIFYPEER"; -string const HttpClientConfig::proxyCaPathKey = "PROXY_CAPATH"; -string const HttpClientConfig::proxyCaInfoKey = "PROXY_CAINFO"; -string const HttpClientConfig::proxyCaInfoValKey = "PROXY_CAINFO_VAL"; +string const ClientConfig::proxySslVerifyHostKey = "PROXY_SSL_VERIFYHOST"; +string const ClientConfig::proxySslVerifyPeerKey = "PROXY_SSL_VERIFYPEER"; +string const ClientConfig::proxyCaPathKey = "PROXY_CAPATH"; +string const ClientConfig::proxyCaInfoKey = "PROXY_CAINFO"; +string const ClientConfig::proxyCaInfoValKey = "PROXY_CAINFO_VAL"; -string const HttpClientConfig::proxyKey = "CURLOPT_PROXY"; -string const HttpClientConfig::noProxyKey = "CURLOPT_NOPROXY"; -string const HttpClientConfig::httpProxyTunnelKey = "CURLOPT_HTTPPROXYTUNNEL"; +string const ClientConfig::proxyKey = "CURLOPT_PROXY"; +string const ClientConfig::noProxyKey = "CURLOPT_NOPROXY"; +string const ClientConfig::httpProxyTunnelKey = "CURLOPT_HTTPPROXYTUNNEL"; -string const HttpClientConfig::connectTimeoutKey = "CONNECTTIMEOUT"; -string const HttpClientConfig::timeoutKey = "TIMEOUT"; -string const HttpClientConfig::lowSpeedLimitKey = "LOW_SPEED_LIMIT"; -string const HttpClientConfig::lowSpeedTimeKey = "LOW_SPEED_TIME"; +string const ClientConfig::connectTimeoutKey = "CONNECTTIMEOUT"; +string const ClientConfig::timeoutKey = "TIMEOUT"; +string const ClientConfig::lowSpeedLimitKey = "LOW_SPEED_LIMIT"; +string const ClientConfig::lowSpeedTimeKey = "LOW_SPEED_TIME"; -string const HttpClientConfig::asyncProcLimitKey = "ASYNC_PROC_LIMIT"; +string const ClientConfig::asyncProcLimitKey = "ASYNC_PROC_LIMIT"; -size_t forwardToHttpClient(char* ptr, size_t size, size_t nmemb, void* userdata) { +size_t forwardToClient(char* ptr, size_t size, size_t nmemb, void* userdata) { size_t const nchars = size * nmemb; - HttpClient* reader = reinterpret_cast(userdata); + Client* reader = reinterpret_cast(userdata); reader->_store(ptr, nchars); return nchars; } -HttpClient::HttpClient(string const& method, string const& url, string const& data, - vector const& headers, HttpClientConfig const& clientConfig) +Client::Client(string const& method, string const& url, string const& data, vector const& headers, + ClientConfig const& clientConfig) : _method(method), _url(url), _data(data), _headers(headers), _clientConfig(clientConfig) { _hcurl = curl_easy_init(); assert(_hcurl != nullptr); // curl_easy_init() failed to allocate memory, etc. } -HttpClient::~HttpClient() { +Client::~Client() { curl_slist_free_all(_hlist); curl_easy_cleanup(_hcurl); } -void HttpClient::read(CallbackType const& onDataRead) { +void Client::read(CallbackType const& onDataRead) { assert(onDataRead != nullptr); // no callback function provided - string const context = "HttpClient::" + string(__func__) + " "; + string const context = "Client::" + string(__func__) + " "; _onDataRead = onDataRead; _errorChecked("curl_easy_setopt(CURLOPT_URL)", curl_easy_setopt(_hcurl, CURLOPT_URL, _url.c_str())); _errorChecked("curl_easy_setopt(CURLOPT_CUSTOMREQUEST)", @@ -173,18 +173,18 @@ void HttpClient::read(CallbackType const& onDataRead) { _errorChecked("curl_easy_setopt(CURLOPT_FAILONERROR)", curl_easy_setopt(_hcurl, CURLOPT_FAILONERROR, 1L)); _errorChecked("curl_easy_setopt(CURLOPT_WRITEFUNCTION)", - curl_easy_setopt(_hcurl, CURLOPT_WRITEFUNCTION, forwardToHttpClient)); + curl_easy_setopt(_hcurl, CURLOPT_WRITEFUNCTION, forwardToClient)); _errorChecked("curl_easy_setopt(CURLOPT_WRITEDATA)", curl_easy_setopt(_hcurl, CURLOPT_WRITEDATA, this)); _errorChecked("curl_easy_perform()", curl_easy_perform(_hcurl)); } -json HttpClient::readAsJson() { +json Client::readAsJson() { vector data; this->read([&data](char const* buf, size_t size) { data.insert(data.cend(), buf, buf + size); }); return json::parse(data); } -void HttpClient::_errorChecked(string const& scope, CURLcode errnum) { +void Client::_errorChecked(string const& scope, CURLcode errnum) { if (errnum != CURLE_OK) { string errorStr = curl_easy_strerror(errnum); long httpResponseCode = 0; @@ -197,6 +197,6 @@ void HttpClient::_errorChecked(string const& scope, CURLcode errnum) { } } -void HttpClient::_store(char const* ptr, size_t nchars) { _onDataRead(ptr, nchars); } +void Client::_store(char const* ptr, size_t nchars) { _onDataRead(ptr, nchars); } -} // namespace lsst::qserv::replica +} // namespace lsst::qserv::http diff --git a/src/replica/HttpClient.h b/src/http/Client.h similarity index 88% rename from src/replica/HttpClient.h rename to src/http/Client.h index 92969e4b15..07e06efb35 100644 --- a/src/replica/HttpClient.h +++ b/src/http/Client.h @@ -18,8 +18,8 @@ * the GNU General Public License along with this program. If not, * see . */ -#ifndef LSST_QSERV_HTTPCLIENT_H -#define LSST_QSERV_HTTPCLIENT_H +#ifndef LSST_QSERV_HTTP_CLIENT_H +#define LSST_QSERV_HTTP_CLIENT_H // System headers #include @@ -31,13 +31,13 @@ #include "nlohmann/json.hpp" // This header declarations -namespace lsst::qserv::replica { +namespace lsst::qserv::http { /** - * Class HttpClientConfig encapsulates configuration parameters related to 'libcurl' + * Class ClientConfig encapsulates configuration parameters related to 'libcurl' * option setter. */ -class HttpClientConfig { +class ClientConfig { public: /// The folder where the parameters are stored in the persistent configuration. static std::string const category; @@ -140,10 +140,10 @@ class HttpClientConfig { // The default state of an object corresponds to not having any of the options // carried by the class be set when using 'libcurl' API. - HttpClientConfig() = default; - HttpClientConfig(HttpClientConfig const&) = default; - HttpClientConfig& operator=(HttpClientConfig const&) = default; - ~HttpClientConfig() = default; + ClientConfig() = default; + ClientConfig(ClientConfig const&) = default; + ClientConfig& operator=(ClientConfig const&) = default; + ~ClientConfig() = default; // Values of the parameters @@ -172,14 +172,14 @@ class HttpClientConfig { }; /** - * Class HttpClient is a simple interface for communicating over the HTTP protocol. + * Class Client is a simple interface for communicating over the HTTP protocol. * The implementation of the class invokes a user-supplied callback (lambda) function for * each sequence of bytes read from the input stream. * * Here is an example of using the class to pull a file and dump its content on * to the standard output stream: * @code - * HttpClient reader("GET", "http://my.host.domain/data/chunk_0.txt"); + * Client reader("GET", "http://my.host.domain/data/chunk_0.txt"); * reader.read([](char const* buf, size_t size) { * std::cout << str::string(buf, size); * }); @@ -193,23 +193,23 @@ class HttpClientConfig { * {"color_id", 123} * }); * std::vector const headers = {"Content-Type: application/json"}; - * HttpClient client("POST", "http://svc.domain.net/create", request.dump(), headers); + * Client client("POST", "http://svc.domain.net/create", request.dump(), headers); * nlohmann::json const result = client.readAsJson(); * @code * */ -class HttpClient { +class Client { public: /// The function type for notifications on each record retrieved from the input stream. typedef std::function CallbackType; // No copy semantics for this class. - HttpClient() = delete; - HttpClient(HttpClient const&) = delete; - HttpClient& operator=(HttpClient const&) = delete; + Client() = delete; + Client(Client const&) = delete; + Client& operator=(Client const&) = delete; /// Non-trivial destructor is needed to free up allocated resources. - ~HttpClient(); + ~Client(); /** * @param method The name of an HTTP method ('GET', 'POST', 'PUT', 'DELETE'). @@ -218,9 +218,9 @@ class HttpClient { * @param headers Optional HTTP headers to be send with a request. * @param clientConfig Optional configuration parameters of the reader. */ - HttpClient(std::string const& method, std::string const& url, std::string const& data = std::string(), - std::vector const& headers = std::vector(), - HttpClientConfig const& clientConfig = HttpClientConfig()); + Client(std::string const& method, std::string const& url, std::string const& data = std::string(), + std::vector const& headers = std::vector(), + ClientConfig const& clientConfig = ClientConfig()); /** * Begin processing a request. The whole content of the remote data source @@ -263,10 +263,10 @@ class HttpClient { * See the implementation of the class for further details on the function. * See the documentation on lincurl C API for an explanation of the function's parameters. */ - friend size_t forwardToHttpClient(char* ptr, size_t size, size_t nmemb, void* userdata); + friend size_t forwardToClient(char* ptr, size_t size, size_t nmemb, void* userdata); /** - * This method is invoked by function forwardToHttpClient() on each chunk of data + * This method is invoked by function forwardToClient() on each chunk of data * reported by CURL while streaming in data from a remote server. * * @param ptr A pointer to the beginning of the data buffer. @@ -280,7 +280,7 @@ class HttpClient { std::string const _url; std::string const _data; std::vector const _headers; - HttpClientConfig const _clientConfig; + ClientConfig const _clientConfig; CallbackType _onDataRead; ///< set by method read() before pulling the data @@ -289,6 +289,6 @@ class HttpClient { curl_slist* _hlist = nullptr; }; -} // namespace lsst::qserv::replica +} // namespace lsst::qserv::http -#endif // LSST_QSERV_HTTPCLIENT_H +#endif // LSST_QSERV_HTTP_CLIENT_H diff --git a/src/replica/HttpExceptions.cc b/src/http/Exceptions.cc similarity index 88% rename from src/replica/HttpExceptions.cc rename to src/http/Exceptions.cc index 4482b32a93..e11e9a675a 100644 --- a/src/replica/HttpExceptions.cc +++ b/src/http/Exceptions.cc @@ -20,7 +20,7 @@ */ // Class header -#include "replica/HttpExceptions.h" +#include "http/Exceptions.h" // System headers #include @@ -28,14 +28,14 @@ using namespace std; using json = nlohmann::json; -namespace lsst::qserv::replica { +namespace lsst::qserv::http { void raiseRetryAllowedError(string const& scope, string const& error, long httpErrCode) { json errorExt; errorExt["http_error"] = httpErrCode; errorExt["system_error"] = errno; errorExt["retry_allowed"] = 1; - throw HttpError(scope, error, errorExt); + throw Error(scope, error, errorExt); } -} // namespace lsst::qserv::replica +} // namespace lsst::qserv::http diff --git a/src/replica/HttpExceptions.h b/src/http/Exceptions.h similarity index 78% rename from src/replica/HttpExceptions.h rename to src/http/Exceptions.h index 519a4af586..0e2638bbd9 100644 --- a/src/replica/HttpExceptions.h +++ b/src/http/Exceptions.h @@ -18,8 +18,8 @@ * the GNU General Public License along with this program. If not, * see . */ -#ifndef LSST_QSERV_HTTPEXCEPTIONS_H -#define LSST_QSERV_HTTPEXCEPTIONS_H +#ifndef LSST_QSERV_HTTP_EXCEPTIONS_H +#define LSST_QSERV_HTTP_EXCEPTIONS_H // System headers #include @@ -29,27 +29,27 @@ #include "nlohmann/json.hpp" // This header declarations -namespace lsst::qserv::replica { +namespace lsst::qserv::http { /** - * Class HttpError represents exceptions thrown by HTTP modules in case of + * Class Error represents exceptions thrown by HTTP modules in case of * any errors which require additional info to be sent back to clients in * response to the requests. */ -class HttpError : public std::runtime_error { +class Error : public std::runtime_error { public: /** * @param func A scope in which the error originated. * @param errorMsg A reason for the exception. * @param errorExt (optional) The additional information on the error. */ - HttpError(std::string const& func, std::string const& errorMsg, - nlohmann::json const& errorExt = nlohmann::json::object()) + Error(std::string const& func, std::string const& errorMsg, + nlohmann::json const& errorExt = nlohmann::json::object()) : std::runtime_error(errorMsg), _func(func), _errorExt(errorExt) {} - HttpError() = default; - HttpError(HttpError const&) = default; - HttpError& operator=(HttpError const&) = default; + Error() = default; + Error(Error const&) = default; + Error& operator=(Error const&) = default; std::string const& func() const { return _func; } nlohmann::json const& errorExt() const { return _errorExt; } @@ -67,10 +67,10 @@ class HttpError : public std::runtime_error { * @param scope A scope of the error. * @param error A human readable error message. * @param (optional) HTTP code of an error if applies. - * @throws HttpError + * @throws Error */ void raiseRetryAllowedError(std::string const& scope, std::string const& error, long httpErrCode = 0); -} // namespace lsst::qserv::replica +} // namespace lsst::qserv::http -#endif // LSST_QSERV_HTTPEXCEPTIONS_H +#endif // LSST_QSERV_HTTP_EXCEPTIONS_H diff --git a/src/replica/Url.cc b/src/http/Url.cc similarity index 98% rename from src/replica/Url.cc rename to src/http/Url.cc index 57ed2ad51e..976b47df29 100644 --- a/src/replica/Url.cc +++ b/src/http/Url.cc @@ -20,7 +20,7 @@ */ // Class header -#include "replica/Url.h" +#include "http/Url.h" // System headers #include @@ -31,7 +31,7 @@ using namespace std; -namespace lsst::qserv::replica { +namespace lsst::qserv::http { Url::Url(string const& url) : _url(url) { _translate(); } @@ -127,4 +127,4 @@ void Url::_translate() { throw invalid_argument(_error(__func__, "invalid url '" + _url + "'")); } -} // namespace lsst::qserv::replica +} // namespace lsst::qserv::http diff --git a/src/replica/Url.h b/src/http/Url.h similarity index 95% rename from src/replica/Url.h rename to src/http/Url.h index a426117f9a..8db08728cf 100644 --- a/src/replica/Url.h +++ b/src/http/Url.h @@ -18,15 +18,15 @@ * the GNU General Public License along with this program. If not, * see . */ -#ifndef LSST_QSERV_URL_H -#define LSST_QSERV_URL_H +#ifndef LSST_QSERV_HTTP_URL_H +#define LSST_QSERV_HTTP_URL_H // System headers #include #include // This header declarations -namespace lsst::qserv::replica { +namespace lsst::qserv::http { /** * Class Url is a helper class for parsing and validating URLs. @@ -110,6 +110,6 @@ class Url { std::string _target; ///< the target part of the url }; -} // namespace lsst::qserv::replica +} // namespace lsst::qserv::http -#endif // LSST_QSERV_URL_H +#endif // LSST_QSERV_HTTP_URL_H diff --git a/src/replica/testHttpAsyncReq.cc b/src/http/testAsyncReq.cc similarity index 83% rename from src/replica/testHttpAsyncReq.cc rename to src/http/testAsyncReq.cc index 444f88136a..434c559307 100644 --- a/src/replica/testHttpAsyncReq.cc +++ b/src/http/testAsyncReq.cc @@ -19,7 +19,7 @@ * see . */ /** - * @brief test HttpAsyncReq + * @brief test AsyncReq */ // System headers @@ -38,34 +38,32 @@ #include "lsst/log/Log.h" // Qserv headers +#include "http/AsyncReq.h" #include "qhttp/Request.h" #include "qhttp/Response.h" #include "qhttp/Server.h" #include "qhttp/Status.h" -#include "replica/HttpAsyncReq.h" -#include "replica/Mutex.h" #include "util/AsyncTimer.h" // Boost unit test header -#define BOOST_TEST_MODULE HttpAsyncReq +#define BOOST_TEST_MODULE AsyncReq #include using namespace std; namespace asio = boost::asio; namespace test = boost::test_tools; -using namespace lsst::qserv::replica; +using namespace lsst::qserv::http; namespace qhttp = lsst::qserv::qhttp; namespace util = lsst::qserv::util; namespace { -class HttpServer { +class Server { public: - HttpServer(HttpServer const&) = delete; - HttpServer& operator=(HttpServer const&) = delete; - explicit HttpServer(uint16_t port = 0) - : _io_service(), _server(qhttp::Server::create(_io_service, port)) {} - ~HttpServer() { + Server(Server const&) = delete; + Server& operator=(Server const&) = delete; + explicit Server(uint16_t port = 0) : _io_service(), _server(qhttp::Server::create(_io_service, port)) {} + ~Server() { // The thread won't be available if the server failed to start due to port // conflict or some other reason. if (_serviceThread != nullptr) { @@ -97,8 +95,8 @@ BOOST_AUTO_TEST_SUITE(Suite) // Test an ability of the class to correctly parse input parameters. -BOOST_AUTO_TEST_CASE(HttpAsyncReq_create) { - LOGS_INFO("HttpAsyncReq_create"); +BOOST_AUTO_TEST_CASE(AsyncReq_create) { + LOGS_INFO("AsyncReq_create"); asio::io_service io_service; @@ -106,7 +104,7 @@ BOOST_AUTO_TEST_CASE(HttpAsyncReq_create) { BOOST_REQUIRE_NO_THROW({ string const url = "http://127.0.0.1:80/"; string const method = "GET"; - auto req = HttpAsyncReq::create(io_service, nullptr, method, url); + auto req = AsyncReq::create(io_service, nullptr, method, url); }); // HTTPS is not supported @@ -114,7 +112,7 @@ BOOST_AUTO_TEST_CASE(HttpAsyncReq_create) { { string const url = "https://127.0.0.1:80/"; string const method = "GET"; - auto req = HttpAsyncReq::create( + auto req = AsyncReq::create( io_service, [](auto const& req) {}, method, url); }, std::invalid_argument); @@ -124,7 +122,7 @@ BOOST_AUTO_TEST_CASE(HttpAsyncReq_create) { { string const url = "http://127.0.0.1:80/"; string const method = "INVALID"; - auto req = HttpAsyncReq::create( + auto req = AsyncReq::create( io_service, [](auto const& req) {}, method, url); }, std::invalid_argument); @@ -133,8 +131,8 @@ BOOST_AUTO_TEST_CASE(HttpAsyncReq_create) { // The simplest test that verifies correct serialization/deserialization // of the header and the body in requests and responses. -BOOST_AUTO_TEST_CASE(HttpAsyncReq_simple) { - LOGS_INFO("HttpAsyncReq_simple"); +BOOST_AUTO_TEST_CASE(AsyncReq_simple) { + LOGS_INFO("AsyncReq_simple"); asio::io_service io_service; @@ -142,14 +140,14 @@ BOOST_AUTO_TEST_CASE(HttpAsyncReq_simple) { // being stuck for longer than expected. auto const testAbortTimer = util::AsyncTimer::create( io_service, chrono::milliseconds(100), [](auto expirationIvalMs) -> bool { - LOGS_INFO("HttpAsyncReq_simple: test exceeded the time budget of " << expirationIvalMs.count() - << "ms"); + LOGS_INFO("AsyncReq_simple: test exceeded the time budget of " << expirationIvalMs.count() + << "ms"); std::exit(1); }); testAbortTimer->start(); // Set up and start the server - ::HttpServer httpServer; + ::Server httpServer; httpServer.server()->addHandler( "GET", "/simple", [](qhttp::Request::Ptr const& req, qhttp::Response::Ptr const& resp) { string const expectedBody = "abcdefg"; @@ -173,11 +171,11 @@ BOOST_AUTO_TEST_CASE(HttpAsyncReq_simple) { string const method = "GET"; string const data = "abcdefg"; unordered_map const headers = {{"Header-1", "A"}, {"Header-2", "B"}}; - shared_ptr const req = HttpAsyncReq::create( + shared_ptr const req = AsyncReq::create( io_service, [testAbortTimer](auto const& req) { testAbortTimer->cancel(); - BOOST_CHECK(req->state() == HttpAsyncReq::State::FINISHED); + BOOST_CHECK(req->state() == AsyncReq::State::FINISHED); BOOST_CHECK(req->errorMessage().empty()); BOOST_CHECK_EQUAL(req->responseCode(), qhttp::STATUS_OK); BOOST_CHECK_EQUAL(req->responseHeader().at("Content-Length"), "0"); @@ -201,12 +199,12 @@ BOOST_AUTO_TEST_CASE(HttpAsyncReq_simple) { // This test is temporary disabled due to changes in the Boost 1.78 (Almalinux 9), where // the following Beast library's method doesn't seem to have any effect: // boost::beast::http::response_parser::body_limit(size_t) -// This isn't critical for Qserv as the below-mentioned status code HttpAsyncReq::State::BODY_LIMIT_ERROR +// This isn't critical for Qserv as the below-mentioned status code AsyncReq::State::BODY_LIMIT_ERROR // is not used by the Replication/Ingest system. // A solution (or a workaround) to this problem will be found later after further investigation. -BOOST_AUTO_TEST_CASE(HttpAsyncReq_body_limit_error) { - LOGS_INFO("HttpAsyncReq_body_limit_error"); +BOOST_AUTO_TEST_CASE(AsyncReq_body_limit_error) { + LOGS_INFO("AsyncReq_body_limit_error"); asio::io_service io_service; @@ -214,14 +212,14 @@ BOOST_AUTO_TEST_CASE(HttpAsyncReq_body_limit_error) { // being stuck for longer than expected. auto const testAbortTimer = util::AsyncTimer::create( io_service, chrono::milliseconds(100), [](auto expirationIvalMs) -> bool { - LOGS_INFO("HttpAsyncReq_body_limit_error: test exceeded the time budget of " + LOGS_INFO("AsyncReq_body_limit_error: test exceeded the time budget of " << expirationIvalMs.count() << "ms"); std::exit(1); }); testAbortTimer->start(); // Set up and start the server - ::HttpServer httpServer; + ::Server httpServer; size_t const serverResponseBodySize = 1024; httpServer.server()->addHandler( "PUT", "/return_large_body", @@ -238,11 +236,11 @@ BOOST_AUTO_TEST_CASE(HttpAsyncReq_body_limit_error) { string const data; unordered_map const headers; size_t const maxResponseBodySize = serverResponseBodySize - 1; - shared_ptr const req = HttpAsyncReq::create( + shared_ptr const req = AsyncReq::create( io_service, [testAbortTimer, serverResponseBodySize](auto const& req) { testAbortTimer->cancel(); - BOOST_CHECK(req->state() == HttpAsyncReq::State::BODY_LIMIT_ERROR); + BOOST_CHECK(req->state() == AsyncReq::State::BODY_LIMIT_ERROR); BOOST_CHECK(req->errorMessage().empty()); BOOST_CHECK_EQUAL(req->responseCode(), qhttp::STATUS_OK); BOOST_CHECK_EQUAL(req->responseHeader().at("Content-Length"), @@ -262,8 +260,8 @@ BOOST_AUTO_TEST_CASE(HttpAsyncReq_body_limit_error) { // Testing request expiration due to non-responsive server (which is simulated // by introducing a delay into the request handler.) -BOOST_AUTO_TEST_CASE(HttpAsyncReq_expired) { - LOGS_INFO("HttpAsyncReq_expired"); +BOOST_AUTO_TEST_CASE(AsyncReq_expired) { + LOGS_INFO("AsyncReq_expired"); asio::io_service io_service; @@ -271,14 +269,14 @@ BOOST_AUTO_TEST_CASE(HttpAsyncReq_expired) { // being stuck for longer than expected. auto const testAbortTimer = util::AsyncTimer::create( io_service, chrono::milliseconds(3000), [](auto expirationIvalMs) -> bool { - LOGS_INFO("HttpAsyncReq_expired: test exceeded the time budget of " - << expirationIvalMs.count() << "ms"); + LOGS_INFO("AsyncReq_expired: test exceeded the time budget of " << expirationIvalMs.count() + << "ms"); std::exit(1); }); testAbortTimer->start(); // Set up and start the server - ::HttpServer httpServer; + ::Server httpServer; httpServer.server()->addHandler("POST", "/delayed_response", [](qhttp::Request::Ptr const& req, qhttp::Response::Ptr const& resp) { this_thread::sleep_for(chrono::milliseconds(2500)); @@ -293,11 +291,11 @@ BOOST_AUTO_TEST_CASE(HttpAsyncReq_expired) { unordered_map const headers; size_t const maxResponseBodySize = 0; unsigned int const expirationIvalSec = 2; - shared_ptr const req = HttpAsyncReq::create( + shared_ptr const req = AsyncReq::create( io_service, [testAbortTimer](auto const& req) { testAbortTimer->cancel(); - BOOST_CHECK(req->state() == HttpAsyncReq::State::EXPIRED); + BOOST_CHECK(req->state() == AsyncReq::State::EXPIRED); BOOST_REQUIRE_NO_THROW(req->errorMessage()); BOOST_CHECK_THROW(req->responseCode(), std::logic_error); BOOST_CHECK_THROW(req->responseHeader(), std::logic_error); @@ -312,8 +310,8 @@ BOOST_AUTO_TEST_CASE(HttpAsyncReq_expired) { // Testing request cancelation for the in-flight request. -BOOST_AUTO_TEST_CASE(HttpAsyncReq_cancelled) { - LOGS_INFO("HttpAsyncReq_cancelled"); +BOOST_AUTO_TEST_CASE(AsyncReq_cancelled) { + LOGS_INFO("AsyncReq_cancelled"); asio::io_service io_service; @@ -321,14 +319,14 @@ BOOST_AUTO_TEST_CASE(HttpAsyncReq_cancelled) { // being stuck for longer than expected. auto const testAbortTimer = util::AsyncTimer::create( io_service, chrono::milliseconds(3000), [](auto expirationIvalMs) -> bool { - LOGS_INFO("HttpAsyncReq_simple: test exceeded the time budget of " << expirationIvalMs.count() - << "ms"); + LOGS_INFO("AsyncReq_simple: test exceeded the time budget of " << expirationIvalMs.count() + << "ms"); std::exit(1); }); testAbortTimer->start(); // Set up and start the server - ::HttpServer httpServer; + ::Server httpServer; httpServer.server()->addHandler("DELETE", "/delayed_response_too", [](qhttp::Request::Ptr const& req, qhttp::Response::Ptr const& resp) { this_thread::sleep_for(chrono::milliseconds(2000)); @@ -339,11 +337,11 @@ BOOST_AUTO_TEST_CASE(HttpAsyncReq_cancelled) { // Submit a request. string const url = "http://127.0.0.1:" + to_string(httpServer.port()) + "/delayed_response_too"; string const method = "DELETE"; - shared_ptr const req = HttpAsyncReq::create( + shared_ptr const req = AsyncReq::create( io_service, [testAbortTimer](auto const& req) { testAbortTimer->cancel(); - BOOST_CHECK(req->state() == HttpAsyncReq::State::CANCELLED); + BOOST_CHECK(req->state() == AsyncReq::State::CANCELLED); }, method, url); req->start(); @@ -362,8 +360,8 @@ BOOST_AUTO_TEST_CASE(HttpAsyncReq_cancelled) { // Testing request cancelation before starting the request. -BOOST_AUTO_TEST_CASE(HttpAsyncReq_cancelled_before_started) { - LOGS_INFO("HttpAsyncReq_cancelled_before_started"); +BOOST_AUTO_TEST_CASE(AsyncReq_cancelled_before_started) { + LOGS_INFO("AsyncReq_cancelled_before_started"); asio::io_service io_service; @@ -371,14 +369,14 @@ BOOST_AUTO_TEST_CASE(HttpAsyncReq_cancelled_before_started) { // being stuck for longer than expected. auto const testAbortTimer = util::AsyncTimer::create( io_service, chrono::milliseconds(300), [](auto expirationIvalMs) -> bool { - LOGS_INFO("HttpAsyncReq_cancelled_before_started: test exceeded the time budget of " + LOGS_INFO("AsyncReq_cancelled_before_started: test exceeded the time budget of " << expirationIvalMs.count() << "ms"); std::exit(1); }); testAbortTimer->start(); // Set up and start the server - ::HttpServer httpServer; + ::Server httpServer; httpServer.server()->addHandler("GET", "/quick", [](qhttp::Request::Ptr const& req, qhttp::Response::Ptr const& resp) { resp->sendStatus(qhttp::STATUS_OK); @@ -388,17 +386,17 @@ BOOST_AUTO_TEST_CASE(HttpAsyncReq_cancelled_before_started) { // Submit a request. string const url = "http://127.0.0.1:" + to_string(httpServer.port()) + "/quick"; string const method = "GET"; - shared_ptr const req = HttpAsyncReq::create( + shared_ptr const req = AsyncReq::create( io_service, [testAbortTimer](auto const& req) { testAbortTimer->cancel(); - BOOST_CHECK(req->state() == HttpAsyncReq::State::CANCELLED); + BOOST_CHECK(req->state() == AsyncReq::State::CANCELLED); }, method, url); // Cancel right away. BOOST_CHECK(req->cancel()); - BOOST_CHECK(req->state() == HttpAsyncReq::State::CANCELLED); + BOOST_CHECK(req->state() == AsyncReq::State::CANCELLED); BOOST_CHECK(!req->cancel()); // since the request was already cancelled // It's not allowed to start the cancelled requests @@ -408,10 +406,10 @@ BOOST_AUTO_TEST_CASE(HttpAsyncReq_cancelled_before_started) { serviceThread.join(); } -// Testing an ability of HttpAsyncReq to wait before the server will start. +// Testing an ability of AsyncReq to wait before the server will start. -BOOST_AUTO_TEST_CASE(HttpAsyncReq_delayed_server_start) { - LOGS_INFO("HttpAsyncReq_delayed_server_start"); +BOOST_AUTO_TEST_CASE(AsyncReq_delayed_server_start) { + LOGS_INFO("AsyncReq_delayed_server_start"); asio::io_service io_service; @@ -423,24 +421,24 @@ BOOST_AUTO_TEST_CASE(HttpAsyncReq_delayed_server_start) { boost::system::error_code ec; socket.bind(boost::asio::ip::tcp::endpoint(boost::asio::ip::tcp::v4(), 0), ec); if (ec) { - LOGS_INFO("HttpAsyncReq_delayed_server_start: bind failed " << ec); + LOGS_INFO("AsyncReq_delayed_server_start: bind failed " << ec); std::exit(1); } uint16_t const port = socket.local_endpoint().port(); - LOGS_INFO("HttpAsyncReq_delayed_server_start: bind port=" << port); + LOGS_INFO("AsyncReq_delayed_server_start: bind port=" << port); // The deadline timer limits the duration of the test to prevent the test from // being stuck for longer than expected. auto const testAbortTimer = util::AsyncTimer::create( io_service, chrono::milliseconds(5000), [](auto expirationIvalMs) -> bool { - LOGS_INFO("HttpAsyncReq_delayed_server_start: test exceeded the time budget of " + LOGS_INFO("AsyncReq_delayed_server_start: test exceeded the time budget of " << expirationIvalMs.count() << "ms"); std::exit(1); }); testAbortTimer->start(); // Set up the server on the allocated port. The server start will be delayed by the timer. - ::HttpServer httpServer(port); + ::Server httpServer(port); httpServer.server()->addHandler("GET", "/redirected_from", [](qhttp::Request::Ptr const& req, qhttp::Response::Ptr const& resp) { resp->headers["Location"] = "/redirected_to"; @@ -448,7 +446,7 @@ BOOST_AUTO_TEST_CASE(HttpAsyncReq_delayed_server_start) { }); // Request object will be created later. - shared_ptr req; + shared_ptr req; // Delay server startup before expiration of the timer auto const serverStartDelayTimer = util::AsyncTimer::create( @@ -461,16 +459,16 @@ BOOST_AUTO_TEST_CASE(HttpAsyncReq_delayed_server_start) { // Submit a request. string const url = "http://127.0.0.1:" + to_string(port) + "/redirected_from"; string const method = "GET"; - req = HttpAsyncReq::create( + req = AsyncReq::create( io_service, [testAbortTimer](auto const& req) { testAbortTimer->cancel(); switch (req->state()) { - case HttpAsyncReq::State::FINISHED: + case AsyncReq::State::FINISHED: BOOST_CHECK_EQUAL(req->responseCode(), qhttp::STATUS_MOVED_PERM); BOOST_CHECK_EQUAL(req->responseHeader().at("Location"), "/redirected_to"); break; - case HttpAsyncReq::State::CANCELLED: + case AsyncReq::State::CANCELLED: break; default: BOOST_CHECK(false); diff --git a/src/replica/testUrl.cc b/src/http/testUrl.cc similarity index 98% rename from src/replica/testUrl.cc rename to src/http/testUrl.cc index ccbcd6a5d2..a3c324185a 100644 --- a/src/replica/testUrl.cc +++ b/src/http/testUrl.cc @@ -25,7 +25,7 @@ #include // Qserv headers -#include "replica/Url.h" +#include "http/Url.h" // Boost unit test header #define BOOST_TEST_MODULE Url @@ -33,7 +33,7 @@ using namespace std; namespace test = boost::test_tools; -using namespace lsst::qserv::replica; +using namespace lsst::qserv::http; BOOST_AUTO_TEST_SUITE(Suite) diff --git a/src/replica/CMakeLists.txt b/src/replica/CMakeLists.txt index 564cd04ed0..557b5d4252 100644 --- a/src/replica/CMakeLists.txt +++ b/src/replica/CMakeLists.txt @@ -150,12 +150,8 @@ target_sources(replica PRIVATE GetStatusQservMgtRequest.h HealthMonitorTask.cc HealthMonitorTask.h - HttpAsyncReq.h - HttpAsyncReq.cc HttpAsyncReqApp.h HttpAsyncReqApp.cc - HttpClient.cc - HttpClient.h HttpClientApp.cc HttpClientApp.h HttpCatalogsModule.cc @@ -166,8 +162,6 @@ target_sources(replica PRIVATE HttpControllersModule.h HttpDirectorIndexModule.cc HttpDirectorIndexModule.h - HttpExceptions.cc - HttpExceptions.h HttpExportModule.cc HttpExportModule.h HttpIngestChunksModule.cc @@ -422,8 +416,6 @@ target_sources(replica PRIVATE TransactionContrib.h TransactionsApp.cc TransactionsApp.h - Url.cc - Url.h VerifyApp.cc VerifyApp.h VerifyJob.cc @@ -469,6 +461,7 @@ target_link_libraries(replica PUBLIC xrdsvc XrdCl XrdSsiLib + http qhttp sphgeom partition @@ -481,8 +474,8 @@ target_link_libraries(replica PUBLIC curl ) -FUNCTION(REPLICA_UTILS) - FOREACH(UTIL IN ITEMS ${ARGV}) +function(replica_utils) + foreach(UTIL in items ${ARGV}) add_executable(${UTIL}) target_sources(${UTIL} PRIVATE tools/${UTIL}.cc) target_include_directories(${UTIL} PRIVATE ${XROOTD_INCLUDE_DIRS}) @@ -490,8 +483,8 @@ FUNCTION(REPLICA_UTILS) replica ) install(TARGETS ${UTIL}) - ENDFOREACH() -ENDFUNCTION() + endforeach() +endfunction() replica_utils( qserv-replica-calc-cs @@ -514,17 +507,17 @@ install( TARGETS replica ) -FUNCTION(REPLICA_TESTS) - FOREACH(TEST IN ITEMS ${ARGV}) +function(replica_tests) + foreach(TEST in items ${argv}) add_executable(${TEST} ${TEST}.cc) target_link_libraries(${TEST} PUBLIC replica Boost::unit_test_framework Threads::Threads ) - add_test(NAME ${TEST} COMMAND ${TEST}) - ENDFOREACH() -ENDFUNCTION() + add_test(name ${TEST} command ${TEST}) + endforeach() +endfunction() replica_tests( testApplicationParser @@ -536,7 +529,6 @@ replica_tests( testCsv testFileIngestApp testFileUtils - testHttpAsyncReq testHttpRequestQuery testIngestRequestMgr testJson @@ -548,6 +540,5 @@ replica_tests( testSqlResultSet testSqlSchemaUtils testTypes - testUrl testConfiguration ) diff --git a/src/replica/HttpAsyncReqApp.cc b/src/replica/HttpAsyncReqApp.cc index adf0d7b439..406a2f5b51 100644 --- a/src/replica/HttpAsyncReqApp.cc +++ b/src/replica/HttpAsyncReqApp.cc @@ -36,7 +36,7 @@ #include "nlohmann/json.hpp" // Qserv headers -#include "replica/HttpAsyncReq.h" +#include "http/AsyncReq.h" using namespace std; using json = nlohmann::json; @@ -134,7 +134,7 @@ int HttpAsyncReqApp::runImpl() { } } boost::asio::io_service io_service; - auto const ptr = HttpAsyncReq::create( + auto const ptr = http::AsyncReq::create( io_service, [this, osPtr](auto const& ptr) { this->_dump(ptr, osPtr); }, _method, _url, _data, headers, _maxResponseBodySize, _expirationIvalSec); @@ -145,16 +145,16 @@ int HttpAsyncReqApp::runImpl() { osPtr->flush(); if (fs.is_open()) fs.close(); } - return ptr->state() == HttpAsyncReq::State::FINISHED ? 0 : 1; + return ptr->state() == http::AsyncReq::State::FINISHED ? 0 : 1; } -void HttpAsyncReqApp::_dump(shared_ptr const& ptr, ostream* osPtr) const { +void HttpAsyncReqApp::_dump(shared_ptr const& ptr, ostream* osPtr) const { if (_verbose) { - cout << "Request completion state: " << HttpAsyncReq::state2str(ptr->state()) + cout << "Request completion state: " << http::AsyncReq::state2str(ptr->state()) << ", error message: " << ptr->errorMessage() << endl; } - if (ptr->state() == HttpAsyncReq::State::FINISHED || - ptr->state() == HttpAsyncReq::State::BODY_LIMIT_ERROR) { + if (ptr->state() == http::AsyncReq::State::FINISHED || + ptr->state() == http::AsyncReq::State::BODY_LIMIT_ERROR) { if (_verbose) { cout << " HTTP response code: " << ptr->responseCode() << "\n"; cout << " response header:\n"; @@ -162,7 +162,7 @@ void HttpAsyncReqApp::_dump(shared_ptr const& ptr, ostream* osPtr) cout << " " << elem.first << ": " << elem.second << "\n"; } } - if (ptr->state() == HttpAsyncReq::State::FINISHED) { + if (ptr->state() == http::AsyncReq::State::FINISHED) { if (_verbose) { cout << " response body size: " << ptr->responseBodySize() << endl; } diff --git a/src/replica/HttpAsyncReqApp.h b/src/replica/HttpAsyncReqApp.h index 0c198ff914..85e15176a8 100644 --- a/src/replica/HttpAsyncReqApp.h +++ b/src/replica/HttpAsyncReqApp.h @@ -28,9 +28,9 @@ #include // Forward declarations -namespace lsst::qserv::replica { -class HttpAsyncReq; -} // namespace lsst::qserv::replica +namespace lsst::qserv::http { +class AsyncReq; +} // namespace lsst::qserv::http // This header declarations namespace lsst::qserv::replica { @@ -40,9 +40,9 @@ namespace lsst::qserv::replica { * the HTTP/HTTPS protocol using the asynchronous client API. If option '--out=' * is present the result will be writted to the specified file. Otherwise the content * will be printed to the standard output stream. - * @note Unlike a similar tool HttpClientApp, this one uses the asynchronous - * API class HttpAsyncReq. - * @see class HttpClient + * @note Unlike a similar tool HttpClientApp, this one uses the asynchronous API. + * @see class http::AsyncReq + * @see class http::Client * @see class HttpClientApp */ class HttpAsyncReqApp : public Application { @@ -73,7 +73,7 @@ class HttpAsyncReqApp : public Application { HttpAsyncReqApp(int argc, char* argv[]); /// Dump results (status, header and body) of a request. - void _dump(std::shared_ptr const& ptr, std::ostream* osPtr) const; + void _dump(std::shared_ptr const& ptr, std::ostream* osPtr) const; std::string _method = "GET"; std::string _url; diff --git a/src/replica/HttpClientApp.cc b/src/replica/HttpClientApp.cc index fe353e2ac1..fc6f22ae4c 100644 --- a/src/replica/HttpClientApp.cc +++ b/src/replica/HttpClientApp.cc @@ -158,7 +158,7 @@ int HttpClientApp::runImpl() { osPtr = &fs; } } - HttpClient reader(_method, _url, _data, headers, _clientConfig); + http::Client reader(_method, _url, _data, headers, _clientConfig); if (_result2json) { if (nullptr != osPtr) *osPtr << reader.readAsJson() << "\n"; } else { diff --git a/src/replica/HttpClientApp.h b/src/replica/HttpClientApp.h index 7830409418..704eb91307 100644 --- a/src/replica/HttpClientApp.h +++ b/src/replica/HttpClientApp.h @@ -22,8 +22,8 @@ #define LSST_QSERV_REPLICA_HTTPCLIENTAPP_H // Qserv headers +#include "http/Client.h" #include "replica/Application.h" -#include "replica/HttpClient.h" // This header declarations namespace lsst::qserv::replica { @@ -65,7 +65,7 @@ class HttpClientApp : public Application { std::string _url; std::string _data; std::string _header; - HttpClientConfig _clientConfig; + http::ClientConfig _clientConfig; std::string _file; bool _result2json = false; bool _silent = false; diff --git a/src/replica/HttpIngestConfigModule.cc b/src/replica/HttpIngestConfigModule.cc index 10b6965b25..1c4e0017fa 100644 --- a/src/replica/HttpIngestConfigModule.cc +++ b/src/replica/HttpIngestConfigModule.cc @@ -27,10 +27,10 @@ // Qserv headers #include "global/stringUtil.h" +#include "http/Client.h" +#include "http/Exceptions.h" #include "replica/Configuration.h" #include "replica/DatabaseServices.h" -#include "replica/HttpClient.h" -#include "replica/HttpExceptions.h" using namespace std; using json = nlohmann::json; @@ -77,55 +77,58 @@ json HttpIngestConfigModule::_get() { auto const getUInt = [&databaseServices, &databaseInfo](json& obj, string const& key) { try { obj[key] = lsst::qserv::stoui( - databaseServices->ingestParam(databaseInfo.name, HttpClientConfig::category, key).value); + databaseServices->ingestParam(databaseInfo.name, http::ClientConfig::category, key) + .value); } catch (DatabaseServicesNotFound const&) { } }; auto const getInt = [&databaseServices, &databaseInfo](json& obj, string const& key) { try { - obj[key] = stoi( - databaseServices->ingestParam(databaseInfo.name, HttpClientConfig::category, key).value); + obj[key] = + stoi(databaseServices->ingestParam(databaseInfo.name, http::ClientConfig::category, key) + .value); } catch (DatabaseServicesNotFound const&) { } }; auto const getLong = [&databaseServices, &databaseInfo](json& obj, string const& key) { try { - obj[key] = stol( - databaseServices->ingestParam(databaseInfo.name, HttpClientConfig::category, key).value); + obj[key] = + stol(databaseServices->ingestParam(databaseInfo.name, http::ClientConfig::category, key) + .value); } catch (DatabaseServicesNotFound const&) { } }; auto const getStr = [&databaseServices, &databaseInfo](json& obj, string const& key) { try { obj[key] = - databaseServices->ingestParam(databaseInfo.name, HttpClientConfig::category, key).value; + databaseServices->ingestParam(databaseInfo.name, http::ClientConfig::category, key).value; } catch (DatabaseServicesNotFound const&) { } }; json result({{"database", databaseInfo.name}}); - getInt(result, HttpClientConfig::sslVerifyHostKey); - getInt(result, HttpClientConfig::sslVerifyPeerKey); - getStr(result, HttpClientConfig::caPathKey); - getStr(result, HttpClientConfig::caInfoKey); - getStr(result, HttpClientConfig::caInfoValKey); + getInt(result, http::ClientConfig::sslVerifyHostKey); + getInt(result, http::ClientConfig::sslVerifyPeerKey); + getStr(result, http::ClientConfig::caPathKey); + getStr(result, http::ClientConfig::caInfoKey); + getStr(result, http::ClientConfig::caInfoValKey); - getInt(result, HttpClientConfig::proxySslVerifyHostKey); - getInt(result, HttpClientConfig::proxySslVerifyPeerKey); - getStr(result, HttpClientConfig::proxyCaPathKey); - getStr(result, HttpClientConfig::proxyCaInfoKey); - getStr(result, HttpClientConfig::proxyCaInfoValKey); + getInt(result, http::ClientConfig::proxySslVerifyHostKey); + getInt(result, http::ClientConfig::proxySslVerifyPeerKey); + getStr(result, http::ClientConfig::proxyCaPathKey); + getStr(result, http::ClientConfig::proxyCaInfoKey); + getStr(result, http::ClientConfig::proxyCaInfoValKey); - getStr(result, HttpClientConfig::proxyKey); - getStr(result, HttpClientConfig::noProxyKey); - getLong(result, HttpClientConfig::httpProxyTunnelKey); + getStr(result, http::ClientConfig::proxyKey); + getStr(result, http::ClientConfig::noProxyKey); + getLong(result, http::ClientConfig::httpProxyTunnelKey); - getLong(result, HttpClientConfig::connectTimeoutKey); - getLong(result, HttpClientConfig::timeoutKey); - getLong(result, HttpClientConfig::lowSpeedLimitKey); - getLong(result, HttpClientConfig::lowSpeedTimeKey); - getUInt(result, HttpClientConfig::asyncProcLimitKey); + getLong(result, http::ClientConfig::connectTimeoutKey); + getLong(result, http::ClientConfig::timeoutKey); + getLong(result, http::ClientConfig::lowSpeedLimitKey); + getLong(result, http::ClientConfig::lowSpeedTimeKey); + getUInt(result, http::ClientConfig::asyncProcLimitKey); return json({{"config", result}}); } @@ -143,7 +146,7 @@ json HttpIngestConfigModule::_update() { auto const update = [&](string const& key, string const& val) { debug(__func__, key + "=" + val); - databaseServices->saveIngestParam(databaseInfo.name, HttpClientConfig::category, key, val); + databaseServices->saveIngestParam(databaseInfo.name, http::ClientConfig::category, key, val); }; auto const updateUInt = [&](string const& key) { if (body().has(key)) update(key, to_string(body().required(key))); @@ -157,27 +160,27 @@ json HttpIngestConfigModule::_update() { auto const updateStr = [&](string const& key) { if (body().has(key)) update(key, body().required(key)); }; - updateInt(HttpClientConfig::sslVerifyHostKey); - updateInt(HttpClientConfig::sslVerifyPeerKey); - updateStr(HttpClientConfig::caPathKey); - updateStr(HttpClientConfig::caInfoKey); - updateStr(HttpClientConfig::caInfoValKey); - - updateInt(HttpClientConfig::proxySslVerifyHostKey); - updateInt(HttpClientConfig::proxySslVerifyPeerKey); - updateStr(HttpClientConfig::proxyCaPathKey); - updateStr(HttpClientConfig::proxyCaInfoKey); - updateStr(HttpClientConfig::proxyCaInfoValKey); - - updateStr(HttpClientConfig::proxyKey); - updateStr(HttpClientConfig::noProxyKey); - updateLong(HttpClientConfig::httpProxyTunnelKey); - - updateLong(HttpClientConfig::connectTimeoutKey); - updateLong(HttpClientConfig::timeoutKey); - updateLong(HttpClientConfig::lowSpeedLimitKey); - updateLong(HttpClientConfig::lowSpeedTimeKey); - updateUInt(HttpClientConfig::asyncProcLimitKey); + updateInt(http::ClientConfig::sslVerifyHostKey); + updateInt(http::ClientConfig::sslVerifyPeerKey); + updateStr(http::ClientConfig::caPathKey); + updateStr(http::ClientConfig::caInfoKey); + updateStr(http::ClientConfig::caInfoValKey); + + updateInt(http::ClientConfig::proxySslVerifyHostKey); + updateInt(http::ClientConfig::proxySslVerifyPeerKey); + updateStr(http::ClientConfig::proxyCaPathKey); + updateStr(http::ClientConfig::proxyCaInfoKey); + updateStr(http::ClientConfig::proxyCaInfoValKey); + + updateStr(http::ClientConfig::proxyKey); + updateStr(http::ClientConfig::noProxyKey); + updateLong(http::ClientConfig::httpProxyTunnelKey); + + updateLong(http::ClientConfig::connectTimeoutKey); + updateLong(http::ClientConfig::timeoutKey); + updateLong(http::ClientConfig::lowSpeedLimitKey); + updateLong(http::ClientConfig::lowSpeedTimeKey); + updateUInt(http::ClientConfig::asyncProcLimitKey); return json::object(); } diff --git a/src/replica/IngestRequest.cc b/src/replica/IngestRequest.cc index 8030fd0b49..45ce6f28c2 100644 --- a/src/replica/IngestRequest.cc +++ b/src/replica/IngestRequest.cc @@ -34,8 +34,8 @@ #include "nlohmann/json.hpp" // Qserv headers +#include "http/Client.h" #include "replica/Configuration.h" -#include "replica/HttpClient.h" #include "replica/HttpExceptions.h" #include "replica/ServiceProvider.h" @@ -285,11 +285,11 @@ IngestRequest::IngestRequest(shared_ptr const& serviceProvider, // as failed for further analysis by the ingest workflows. try { IngestRequest::_validateState(trans, database, _contrib); - _resource.reset(new Url(_contrib.url)); + _resource.reset(new http::Url(_contrib.url)); switch (_resource->scheme()) { - case Url::FILE: - case Url::HTTP: - case Url::HTTPS: + case http::Url::FILE: + case http::Url::HTTP: + case http::Url::HTTPS: break; default: throw invalid_argument(context + "unsupported url '" + _contrib.url + "'"); @@ -309,7 +309,7 @@ IngestRequest::IngestRequest(shared_ptr const& serviceProvider, : IngestFileSvc(serviceProvider, workerName), _contrib(contrib) { // This constructor assumes a valid contribution object obtained from a database // was passed into the method. - _resource.reset(new Url(_contrib.url)); + _resource.reset(new http::Url(_contrib.url)); _dialect = csv::Dialect(_contrib.dialectInput); } @@ -439,11 +439,11 @@ void IngestRequest::_processReadData() { } try { switch (_resource->scheme()) { - case Url::FILE: + case http::Url::FILE: _readLocalFile(lock); break; - case Url::HTTP: - case Url::HTTPS: + case http::Url::HTTP: + case http::Url::HTTPS: _readRemoteFile(lock); break; default: @@ -599,8 +599,8 @@ void IngestRequest::_readRemoteFile(replica::Lock const& lock) { // Read and parse data from the data source auto parser = make_unique(_dialect); bool const flush = true; - HttpClient reader(_contrib.httpMethod, _contrib.url, _contrib.httpData, _contrib.httpHeaders, - clientConfig); + http::Client reader(_contrib.httpMethod, _contrib.url, _contrib.httpData, _contrib.httpHeaders, + clientConfig); reader.read([&](char const* record, size_t size) { parser->parse(record, size, !flush, reportRow); _contrib.numBytes += size; @@ -610,11 +610,11 @@ void IngestRequest::_readRemoteFile(replica::Lock const& lock) { parser->parse(emptyRecord.data(), emptyRecord.size(), flush, reportRow); } -HttpClientConfig IngestRequest::_clientConfig(replica::Lock const& lock) const { +http::ClientConfig IngestRequest::_clientConfig(replica::Lock const& lock) const { auto const databaseServices = serviceProvider()->databaseServices(); auto const getString = [&](string& val, string const& key) -> bool { try { - val = databaseServices->ingestParam(_contrib.database, HttpClientConfig::category, key).value; + val = databaseServices->ingestParam(_contrib.database, http::ClientConfig::category, key).value; } catch (DatabaseServicesNotFound const&) { return false; } @@ -628,24 +628,24 @@ HttpClientConfig IngestRequest::_clientConfig(replica::Lock const& lock) const { string str; if (getString(str, key)) val = stol(str); }; - HttpClientConfig clientConfig; - getBool(clientConfig.sslVerifyHost, HttpClientConfig::sslVerifyHostKey); - getBool(clientConfig.sslVerifyPeer, HttpClientConfig::sslVerifyPeerKey); - getString(clientConfig.caPath, HttpClientConfig::caPathKey); - getString(clientConfig.caInfo, HttpClientConfig::caInfoKey); - getString(clientConfig.caInfoVal, HttpClientConfig::caInfoValKey); - getBool(clientConfig.proxySslVerifyHost, HttpClientConfig::proxySslVerifyHostKey); - getBool(clientConfig.proxySslVerifyPeer, HttpClientConfig::proxySslVerifyPeerKey); - getString(clientConfig.proxyCaPath, HttpClientConfig::proxyCaPathKey); - getString(clientConfig.proxyCaInfo, HttpClientConfig::proxyCaInfoKey); - getString(clientConfig.proxyCaInfoVal, HttpClientConfig::proxyCaInfoValKey); - getString(clientConfig.proxy, HttpClientConfig::proxyKey); - getString(clientConfig.noProxy, HttpClientConfig::noProxyKey); - getLong(clientConfig.httpProxyTunnel, HttpClientConfig::httpProxyTunnelKey); - getLong(clientConfig.connectTimeout, HttpClientConfig::connectTimeoutKey); - getLong(clientConfig.timeout, HttpClientConfig::timeoutKey); - getLong(clientConfig.lowSpeedLimit, HttpClientConfig::lowSpeedLimitKey); - getLong(clientConfig.lowSpeedTime, HttpClientConfig::lowSpeedTimeKey); + http::ClientConfig clientConfig; + getBool(clientConfig.sslVerifyHost, http::ClientConfig::sslVerifyHostKey); + getBool(clientConfig.sslVerifyPeer, http::ClientConfig::sslVerifyPeerKey); + getString(clientConfig.caPath, http::ClientConfig::caPathKey); + getString(clientConfig.caInfo, http::ClientConfig::caInfoKey); + getString(clientConfig.caInfoVal, http::ClientConfig::caInfoValKey); + getBool(clientConfig.proxySslVerifyHost, http::ClientConfig::proxySslVerifyHostKey); + getBool(clientConfig.proxySslVerifyPeer, http::ClientConfig::proxySslVerifyPeerKey); + getString(clientConfig.proxyCaPath, http::ClientConfig::proxyCaPathKey); + getString(clientConfig.proxyCaInfo, http::ClientConfig::proxyCaInfoKey); + getString(clientConfig.proxyCaInfoVal, http::ClientConfig::proxyCaInfoValKey); + getString(clientConfig.proxy, http::ClientConfig::proxyKey); + getString(clientConfig.noProxy, http::ClientConfig::noProxyKey); + getLong(clientConfig.httpProxyTunnel, http::ClientConfig::httpProxyTunnelKey); + getLong(clientConfig.connectTimeout, http::ClientConfig::connectTimeoutKey); + getLong(clientConfig.timeout, http::ClientConfig::timeoutKey); + getLong(clientConfig.lowSpeedLimit, http::ClientConfig::lowSpeedLimitKey); + getLong(clientConfig.lowSpeedTime, http::ClientConfig::lowSpeedTimeKey); return clientConfig; } diff --git a/src/replica/IngestRequest.h b/src/replica/IngestRequest.h index 6a5b6c4181..07207be06f 100644 --- a/src/replica/IngestRequest.h +++ b/src/replica/IngestRequest.h @@ -29,15 +29,19 @@ #include // Qserv headers +#include "http/Url.h" #include "replica/Csv.h" #include "replica/DatabaseServices.h" #include "replica/IngestFileSvc.h" -#include "replica/Url.h" #include "replica/Mutex.h" // Forward declarations + +namespace lsst::qserv::http { +class ClientConfig; +} // namespace lsst::qserv::http + namespace lsst::qserv::replica { -class HttpClientConfig; class ServiceProvider; } // namespace lsst::qserv::replica @@ -228,7 +232,7 @@ class IngestRequest : public std::enable_shared_from_this, public * Pull file reader's configuration from the config store. * @return The configuration object. */ - HttpClientConfig _clientConfig(replica::Lock const& lock) const; + http::ClientConfig _clientConfig(replica::Lock const& lock) const; /// Mutex guarding internal state. mutable replica::Mutex _mtx; @@ -238,7 +242,7 @@ class IngestRequest : public std::enable_shared_from_this, public TransactionContribInfo _contrib; // These variables are set after completing parameter validation - std::unique_ptr _resource; + std::unique_ptr _resource; csv::Dialect _dialect; /// The flag is set by method process(), and once it's set it's never diff --git a/src/replica/IngestResourceMgrP.cc b/src/replica/IngestResourceMgrP.cc index 605739d52f..42fbef01f0 100644 --- a/src/replica/IngestResourceMgrP.cc +++ b/src/replica/IngestResourceMgrP.cc @@ -24,8 +24,8 @@ // Qserv headers #include "global/stringUtil.h" +#include "http/Client.h" #include "replica/DatabaseServices.h" -#include "replica/HttpClient.h" #include "replica/ServiceProvider.h" using namespace std; @@ -45,8 +45,8 @@ unsigned int IngestResourceMgrP::asyncProcLimit(string const& databaseName) cons auto const databaseServices = _serviceProvider->databaseServices(); try { string const str = databaseServices - ->ingestParam(databaseName, HttpClientConfig::category, - HttpClientConfig::asyncProcLimitKey) + ->ingestParam(databaseName, http::ClientConfig::category, + http::ClientConfig::asyncProcLimitKey) .value; return lsst::qserv::stoui(str); } catch (DatabaseServicesNotFound const&) { diff --git a/src/replica/IngestSvcConn.cc b/src/replica/IngestSvcConn.cc index 79cd829d51..b8259e863e 100644 --- a/src/replica/IngestSvcConn.cc +++ b/src/replica/IngestSvcConn.cc @@ -36,11 +36,11 @@ // Qserv headers #include "global/constants.h" +#include "http/Url.h" #include "replica/Configuration.h" #include "replica/HttpExceptions.h" #include "replica/ReplicaInfo.h" #include "replica/ServiceProvider.h" -#include "replica/Url.h" // LSST headers #include "lsst/log/Log.h" @@ -191,8 +191,8 @@ void IngestSvcConn::_handshakeReceived(boost::system::error_code const& ec, size csv::Dialect dialect; try { - Url const resource(_contrib.url); - if (resource.scheme() != Url::FILE) { + http::Url const resource(_contrib.url); + if (resource.scheme() != http::Url::FILE) { throw invalid_argument(context + string(__func__) + " unsupported url '" + _contrib.url + "'"); } dialect = csv::Dialect(_contrib.dialectInput); diff --git a/src/replica/Registry.cc b/src/replica/Registry.cc index 82df3f1479..2edd4b85d6 100644 --- a/src/replica/Registry.cc +++ b/src/replica/Registry.cc @@ -23,9 +23,9 @@ #include "replica/Registry.h" // Qserv headers +#include "http/Client.h" #include "replica/Configuration.h" #include "replica/ConfigWorker.h" -#include "replica/HttpClient.h" #include "util/common.h" // LSST headers @@ -129,7 +129,7 @@ json Registry::_request(string const& method, string const& resource, json const string const url = _baseUrl + resource; vector const headers = request.empty() ? vector({}) : vector({"Content-Type: application/json"}); - HttpClient client(method, url, request.empty() ? string() : request.dump(), headers); + http::Client client(method, url, request.empty() ? string() : request.dump(), headers); json const response = client.readAsJson(); if (0 == response.at("success").get()) { string const msg = ::context(__func__) + "'" + method + "' request to '" + url +