refactor: Move logging into its own lib

Also moved a few logging deps from lib_spark -> system:
* bts_chrono.h
* bts_no_copy.h
* defer.h
* dispatcher.h/cpp
* file_utils.h/cpp
* type_name.h
This commit is contained in:
John Colvin committed 2024-09-18 14:02:40 +00:00
1 parent 49c7683d58
commit 1b9c4fba2f
235 files changed
+1026 -875

No files matched your search

+1
View File
@@ -107,6 +107,7 @@ else()
add_subdirectory(src/lib_system)
add_subdirectory(src/lib_domain)
add_subdirectory(src/lib_python_md)
add_subdirectory(src/lib_logging)
add_subdirectory(src/app_sparkd)
add_subdirectory(src/data_files)
endif()
@@ -1,22 +1,22 @@
#pragma once
#include "logging/logging.h"
#include "spark/api/public/ServiceContext.h"
#include "spark/api/services/AlgoDefinitionService.server.pb.h"
#include "spark/api/services/OrderService.client.pb.h"
#include "spark/api/services/OrderService.pb.h"
#include "spark/logging/logging.h"
#include "spark/persistence/IStore.h"
#include "spark/persistence/SqliteStore.h"
#include "spark/utils/PBUtils.h"
#include "spark/utils/TimeStamp.h"
#include "spark/utils/bts_assert.h"
#include "spark/utils/bts_no_copy.h"
#include <memory>
#include <boost/tokenizer.hpp>
#include <fmt/format.h>
#include <google/protobuf/util/time_util.h>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <utility>
@@ -69,4 +69,4 @@ class AlgoDefinitionServiceServer : public AlgoDefinitionServiceBase {
} // namespace algo
} // namespace orderman
} // namespace api
} // namespace bts
} // namespace bts
@@ -1,8 +1,8 @@
#include "AnalyticsServiceServer.h"
#include "logging/logging.h"
#include "spark/api/apicore.pb.h"
#include "spark/api/services/AnalyticsService.pb.h"
#include "spark/api/services/AnalyticsService.server.pb.h"
#include "spark/logging/logging.h"
#include "spark/persistence/IStore.h"
#include "spark/persistence/SqliteStore.h"
#include "spark/utils/PBUtils.h"
+4 -4
View File
@@ -1,9 +1,9 @@
#include "AdminReport.h"
#include "logging/logging.h"
#include "logging/protobufs/logging.pb.h"
#include "spark/api/public/ServiceContext.h"
#include "spark/config/config.h"
#include "spark/config/protos/logging.pb.h"
#include "spark/logging/logging.h"
#include "spark/quickbus/quickbus.h"
#include "spark/api/service_helpers/AdminServiceUtils.h"
@@ -17,9 +17,9 @@
namespace bts::cmds {
void md_report() {
auto cfg = config::get<config::Logging>();
auto cfg = config::get<logging::protobufs::Logging>();
cfg.mutable_console()->set_default_level("Fatal");
log::configure_logging(cfg);
logging::configure_logging(cfg);
std::map<api::CMEMics, std::string> mic_map;
mic_map[api::CMEMics::CME] = "XCME";
+1 -1
View File
@@ -1,5 +1,5 @@
#include "CMECert.h"
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include "spark/orderman/ilink3/IL3Dispatcher.h"
#include "spark/orderman/ilink3/IL3GW.h"
#include "spark/orderman/ilink3/IL3Reactor.h"
+1 -1
View File
@@ -1,8 +1,8 @@
#include "KeyCmd.h"
#include "logging/logging.h"
#include "spark/config/config.h"
#include "spark/config/protos/keycode.pb.h"
#include "spark/logging/logging.h"
#include "spark/os/os.h"
#include "spark/utils/AuthToken.h"
#include "spark/utils/FileUtils.h"
+1 -1
View File
@@ -1,5 +1,5 @@
#include "QuickTest.h"
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include <thread>
+1 -1
View File
@@ -1,9 +1,9 @@
#include "Run.h"
#include "logging/logging.h"
#include "spark/api/public/ServiceContext.h"
#include "spark/config/config.h"
#include "spark/config/protos/ports.pb.h"
#include "spark/config/protos/quickbus_cfg.pb.h"
#include "spark/logging/logging.h"
#include "spark/md/MBOProcessor.h"
#include "spark/md/MBOServiceServer.h"
#include "spark/md/MDClient.h"
@@ -1,7 +1,7 @@
#include "TestPythonMdProviderCmd.h"
#include "logging/logging.h"
#include "python_md/md_sink_impl.h"
#include "python_md/py_md_executor.h"
#include "spark/logging/logging.h"
#include <pybind11/embed.h>
#include <pybind11/pybind11.h>
@@ -1,5 +1,5 @@
#include "DeribitDemo.h"
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include "sparkd/cmds/crypto/DeribitDemoListener.h"
#include <csignal>
@@ -1,6 +1,6 @@
#include "DeribitDemoConfiguration.h"
#include "RapidJsonReader.h"
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include "rapidjson/istreamwrapper.h"
#include <emumba-deribit-connector/deribit_utils.h>
#include <fstream>
@@ -1,9 +1,9 @@
#ifdef SPARK_ENABLE_CCAPI
#include "LiveLadderDemoCmd.h"
#include "LiveLadderDemo.h"
#include "logging/logging.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/io/ProtoReader.h"
#include "spark/logging/logging.h"
#include "sparkd/cmds/common/Validators.h"
#include "sparkd/cmds/crypto/LiveLadderDemo.h"
#include <CLI11/CLI11.hpp>
@@ -1,7 +1,7 @@
#include "Cleanup.h"
#include "logging/logging.h"
#include "spark/config/config.h"
#include "spark/config/protos/market_data.pb.h"
#include "spark/logging/logging.h"
#include "spark/md/ShmSegmentName.h"
#include "spark/md/protos/co2.pb.h"
#include <boost/filesystem.hpp>
+1 -1
View File
@@ -1,8 +1,8 @@
#include "Run.h"
#include "Cleanup.h"
#include "domain/security_criteria.h"
#include "logging/logging.h"
#include "spark/config/protos/market_data.pb.h"
#include "spark/logging/logging.h"
#include "spark/md/SymbolLoader.h"
#include "spark/md/UserSymbol.h"
#include "spark/md/relay/RelayRecorder.h"
@@ -10,11 +10,11 @@
#include <iostream>
#include <string>
#include "logging/logging.h"
#include "spark/config/config.h"
#include "spark/config/protos/exchange_sim_cfg.pb.h"
#include "spark/config/protos/ports.pb.h"
#include "spark/domain_model/Security.pb.h"
#include "spark/logging/logging.h"
#include "spark/md/PriceSize.h"
#include "spark/os/os.h"
#include "system/protobufs/system_config.pb.h"
@@ -2,10 +2,10 @@
#include "spark/md/SecurityDB.h"
#include "IDGenerator.h"
#include "logging/logging.h"
#include "spark/config/config.h"
#include "spark/config/protos/exchange_sim_cfg.pb.h"
#include "spark/domain_model/Security.h"
#include "spark/logging/logging.h"
#include "spark/md/MBOProcessor.h"
#include "spark/md/MDProducer.h"
#include "spark/md/Shm/FixedSecurity.h"
@@ -1,11 +1,11 @@
#include "EMExecutionGateway.h"
#include "logging/logging.h"
#include "spark/api/service_helpers/AdminServiceUtils.h"
#include "spark/api/services/AdminService.pb.h"
#include "spark/api/services/OrderService.pb.h"
#include "spark/domain_model/Security.pb.h"
#include "spark/domain_model/SecurityKey.h"
#include "spark/domain_model/common.h"
#include "spark/logging/logging.h"
#include "spark/md/SecurityOracle.h"
#include "spark/orderman/CancelOrderMsg.h"
#include "spark/orderman/CancelReplaceOrderMsg.h"
+1 -1
View File
@@ -1,10 +1,10 @@
#include "ApiGateway.h"
#include "logging/logging.h"
#include "spark/api/core/ApiClientInterop.h"
#include "spark/api/public/ApiUtils.h"
#include "spark/api/services/AdminService.client.pb.h"
#include "spark/api/services/AuthService.client.pb.h"
#include "spark/logging/logging.h"
#include "spark/utils/bts_assert.h"
#include <mutex>
#include <unordered_map>
@@ -1,5 +1,6 @@
#include "MultipartFormDecoder.h"
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string/iter_find.hpp>
+1 -1
View File
@@ -3,9 +3,9 @@
#include "WebServer.h"
#include "ApiGateway.h"
#include "MultipartFormDecoder.h"
#include "logging/logging.h"
#include "spark/api/apigateway.pb.h"
#include "spark/api/core/ApiClientInterop.h"
#include "spark/logging/logging.h"
#include "spark/os/os.h"
#include "spark/tracing/PerfLog.h"
#include "spark/utils/Dispatcher.h"
@@ -1,11 +1,11 @@
#include "MarketDataAuxiliaryServiceServer.h"
#include "VolumeAtPriceState.h"
#include "logging/logging.h"
#include "spark/api/public/ServiceContext.h"
#include "spark/api/services/MarketDataRelay.client.pb.h"
#include "spark/api/services/MarketDataRelay.pb.h"
#include "spark/config/config.h"
#include "spark/config/protos/toggles.pb.h"
#include "spark/logging/logging.h"
#include "spark/utils/Dispatcher.h"
#include "spark/utils/PBUtils.h"
#include "spark/utils/bts_chrono.h"
@@ -1,10 +1,10 @@
#include "MarketDataGateway.h"
#include "domain/security_id.h"
#include "logging/logging.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/api/md/MarketDataConfigUtils.h"
#include "spark/config/config.h"
#include "spark/domain/DomainSecurityUtils.h"
#include "spark/logging/logging.h"
#include "spark/md/GeneralMarketDataListener.h"
#include "spark/md/GeneralSecurityIndex.h"
#include "spark/md/IMarketDataListener.h"
@@ -7,13 +7,13 @@
#include "domain/security_id.h"
#include "domain/spot_security.h"
#include "domain/trade_record.h"
#include "logging/logging.h"
#include "python_md/md_sink.h"
#include "python_md/py_book.h"
#include "python_md/py_md_executor.h"
#include "python_md/py_security.h"
#include "python_md/py_security_list.h"
#include "python_md/py_trade.h"
#include "spark/logging/logging.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/os/os.h"
#include "spark/persistence/IDGenerator.h"
@@ -1,6 +1,6 @@
#include "ReplayMarketDataSubscription.h"
#include "logging/logging.h"
#include "spark/api/md/MarketDataReplayConfig.pb.h"
#include "spark/logging/logging.h"
#include "spark/md/GeneralMarketDataListener.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/md/MDProducer.h"
@@ -1,6 +1,6 @@
#pragma once
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include "sparkd/md/binance/BinanceFilterType.h"
#include "system/json/json_reader.h"
@@ -3,7 +3,7 @@
#include "domain/protobufs/venue.pb.h"
#include "domain/security.h"
#include "domain/security_criteria.h"
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/utils/bts_fatal.h"
#include "sparkd/md/binance/BinanceMessageType.h"
@@ -2,7 +2,7 @@
#include "domain/protobufs/venue.pb.h"
#include "domain/quantity.h"
#include "domain/security.h"
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include "system/json/json_reader.h"
#include <ccapi_cpp/ccapi_message.h>
@@ -2,10 +2,10 @@
#include "domain/protobufs/venue.pb.h"
#include "domain/security.h"
#include "domain/security_criteria.h"
#include "logging/logging.h"
#include "spark/api/md/BinanceMarketDataConfig.pb.h"
#include "spark/io/ProtoPrinter.h"
#include "spark/io/QueryParams.h"
#include "spark/logging/logging.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/binance/BinanceMessageType.h"
@@ -3,7 +3,7 @@
#include "domain/protobufs/venue.pb.h"
#include "domain/security.h"
#include "domain/security_criteria.h"
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/binance_us/BinanceUsMessageType.h"
#include "sparkd/md/binance_us/BinanceUsSpot.h"
@@ -2,10 +2,10 @@
#include "domain/protobufs/venue.pb.h"
#include "domain/security.h"
#include "domain/security_criteria.h"
#include "logging/logging.h"
#include "spark/api/md/BinanceUsMarketDataConfig.pb.h"
#include "spark/io/ProtoPrinter.h"
#include "spark/io/QueryParams.h"
#include "spark/logging/logging.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/binance/BinanceSpot.h"
@@ -2,7 +2,7 @@
#include "domain/price.h"
#include "domain/quantity.h"
#include "domain/security.h"
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include "system/json/json_dumper.h"
#include "system/json/json_reader.h"
@@ -2,8 +2,8 @@
#include "domain/security.h"
#include "domain/security_criteria.h"
#include "domain/security_id.h"
#include "logging/logging.h"
#include "spark/api/md/BinanceUsdsFuturesMarketDataConfig.pb.h"
#include "spark/logging/logging.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/utils/bts_fatal.h"
#include "sparkd/md/binance_usds_futures/BinanceUsdsFuture.h"
@@ -3,10 +3,10 @@
#include "domain/security.h"
#include "domain/security_criteria.h"
#include "domain/security_id.h"
#include "logging/logging.h"
#include "spark/api/md/BinanceUsdsFuturesMarketDataConfig.pb.h"
#include "spark/io/ProtoPrinter.h"
#include "spark/io/QueryParams.h"
#include "spark/logging/logging.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesMessageType.h"
@@ -1,6 +1,6 @@
#pragma once
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include <ccapi_cpp/ccapi_event_handler.h>
@@ -1,9 +1,9 @@
#include "CCApiMarketDataProvider.h"
#include "domain/protobufs/venue.pb.h"
#include "logging/logging.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/config/config.h"
#include "spark/io/ProtoPrinter.h"
#include "spark/logging/logging.h"
#include "spark/md/GeneralMarketDataListener.h"
#include "spark/md/GeneralSecurityIndex.h"
#include "spark/md/MDProducer.h"
@@ -1,5 +1,5 @@
#include "CCApiSession.h"
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include "sparkd/md/ccapi/CCApiCorrelationID.h"
#include "sparkd/md/ccapi/CCApiEventDemuxer.h"
#include "sparkd/md/ccapi/ICCApiMessageHandler.h"
@@ -1,5 +1,5 @@
#include "CCDemoListener.h"
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include "sparkd/md/MarketDataError.h"
#include <functional>
@@ -3,7 +3,7 @@
#include "domain/protobufs/venue.pb.h"
#include "domain/security.h"
#include "domain/security_criteria.h"
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/ccapi/CCApiCorrelationID.h"
#include "sparkd/md/ccapi/CCApiMessageHandler.h"
@@ -2,7 +2,7 @@
#include "domain/protobufs/currency.pb.h"
#include "domain/quantity.h"
#include "domain/security.h"
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include "system/json/json_reader.h"
#include <ccapi_cpp/ccapi_message.h>
@@ -2,7 +2,7 @@
#include "domain/book.h"
#include "domain/book_level_comp.h"
#include "domain/security_id.h"
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/utils/bts_assert.h"
@@ -1,7 +1,7 @@
#include "DeribitInstrumentStore.h"
#include "domain/protobufs/security_type.h"
#include "domain/security.h"
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/utils/bts_fatal.h"
#include "sparkd/md/deribit/DeribitCombo.h"
@@ -60,10 +60,10 @@
#include "domain/protobufs/venue.pb.h"
#include "domain/security.h"
#include "domain/security_criteria.h"
#include "logging/logging.h"
#include "spark/io/websocket/JsonRpcError.h"
#include "spark/io/websocket/JsonRpcHandler.h"
#include "spark/io/websocket/JsonRpcRequest.h"
#include "spark/logging/logging.h"
#include "sparkd/md/ccapi/CCApiCorrelationID.h"
#include "sparkd/md/ccapi/CCApiJsonRpcRequest.h"
#include "sparkd/md/ccapi/CCApiJsonRpcRouter.h"
@@ -2,12 +2,12 @@
#include "domain/protobufs/venue.pb.h"
#include "domain/security.h"
#include "domain/security_criteria.h"
#include "logging/logging.h"
#include "spark/api/md/DeribitMarketDataConfig.pb.h"
#include "spark/domain_model/Security.pb.h"
#include "spark/io/ProtoPrinter.h"
#include "spark/io/websocket/JsonRpcError.h"
#include "spark/io/websocket/JsonRpcRequest.h"
#include "spark/logging/logging.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/ccapi/CCApiCorrelationID.h"
+1 -1
View File
@@ -2,7 +2,7 @@
#include "domain/future_security.h"
#include "domain/protobufs/venue.pb.h"
#include "domain/security.h"
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include "system/json/json_reader.h"
#include "system/table_formatter.h"
+1 -1
View File
@@ -1,7 +1,7 @@
#include "FTXMarket.h"
#include "domain/protobufs/venue.pb.h"
#include "domain/security.h"
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include "spark/utils/bts_fatal.h"
#include "sparkd/md/ftx/FTXSecurityType.h"
#include "system/json/json_reader.h"
@@ -3,7 +3,7 @@
#include "domain/protobufs/venue.pb.h"
#include "domain/security.h"
#include "domain/security_criteria.h"
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/ccapi/CCApiCorrelationID.h"
#include "sparkd/md/ccapi/CCApiMessageHandler.h"
@@ -2,9 +2,9 @@
#include "domain/protobufs/venue.pb.h"
#include "domain/security.h"
#include "domain/security_criteria.h"
#include "logging/logging.h"
#include "spark/api/md/FTXMarketDataConfig.pb.h"
#include "spark/io/ProtoPrinter.h"
#include "spark/logging/logging.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/ccapi/CCApiCorrelationID.h"
@@ -1,8 +1,8 @@
#include "OnixsGlbxSubscription.h"
#include "logging/logging.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/api/md/MarketDataConfigUtils.h"
#include "spark/config/config.h"
#include "spark/logging/logging.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/md/MBOProcessor.h"
#include "spark/md/SymbolLoader.h"
@@ -2,7 +2,7 @@
#include "domain/book.h"
#include "domain/book_level_comp.h"
#include "domain/security_id.h"
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include "spark/md/IMarketDataListener.h"
#include "sparkd/md/ccapi/CCApiCorrelationID.h"
#include "system/json/json_reader.h"
@@ -1,7 +1,7 @@
#include "OkxInstrumentStore.h"
#include "domain/protobufs/security_type.h"
#include "domain/security.h"
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/utils/bts_fatal.h"
#include "sparkd/md/okx/OkxInstrument.h"
@@ -2,9 +2,9 @@
#include "domain/protobufs/venue.pb.h"
#include "domain/security.h"
#include "domain/security_criteria.h"
#include "logging/logging.h"
#include "spark/api/md/OkxMarketDataConfig.pb.h"
#include "spark/io/ProtoPrinter.h"
#include "spark/logging/logging.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/ccapi/CCApiSession.h"
@@ -1,8 +1,8 @@
#include "EmMDListener.h"
#include "domain/protobufs/trading_status.pb.h"
#include "logging/logging.h"
#include "spark/api/services/ShellDiagnosticService.server.pb.h"
#include "spark/domain_model/Security.pb.h"
#include "spark/logging/logging.h"
#include "spark/md/GeneralSecurityIndex.h"
#include "spark/md/IntPrice.h"
#include "spark/md/MDProducer.h"
@@ -1,8 +1,8 @@
#include "EmumbaMain.h"
#include "EMTradingSession.h"
#include "EmMDListener.h"
#include "logging/logging.h"
#include "spark/config/config.h"
#include "spark/logging/logging.h"
#include "spark/md/GeneralSecurityIndex.h"
#include "spark/md/MDProducer.h"
#include "spark/md/SecurityDB.h"
@@ -1,9 +1,11 @@
#pragma once
#include "logging/logging.h"
#include "spark/domain_model/Security.pb.h"
#include "spark/logging/logging.h"
#include "spark/md/common.h"
#include "spark/orderman/RFQMsg.h"
#include "spark/os/os.h"
#include <boost/container/small_vector.hpp>
#include <emumba-deribit-connector/deribit_handler.h>
#include <emumba-deribit-connector/deribit_listener.h>
@@ -1,8 +1,8 @@
#include "MDP3Main.h"
#include "domain/protobufs/trading_status.pb.h"
#include "logging/logging.h"
#include "spark/config/config.h"
#include "spark/logging/logging.h"
#include "spark/md/MBOProcessor.h"
#include "spark/md/SecurityDB.h"
#include "spark/md/cme/MDP3Utils.h"
@@ -1,11 +1,11 @@
#include "MDRecorder.h"
#include "logging/logging.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/api/md/MarketDataConfigUtils.h"
#include "spark/api/services/MarketDataRelay.client.pb.h"
#include "spark/api/services/MarketDataRelay.pb.h"
#include "spark/config/config.h"
#include "spark/domain_model/Security.pb.h"
#include "spark/logging/logging.h"
#include "spark/md/MDClient.h"
#include "spark/md/MDConsumer.h"
#include "spark/md/MDConsumerHandle.h"
@@ -8,9 +8,9 @@
#include "domain/security_market_data_statistics.h"
#include "domain/side.h"
#include "domain/trade_record.h"
#include "logging/logging.h"
#include "spark/domain/DomainSecurityUtils.h"
#include "spark/domain_model/Security.pb.h"
#include "spark/logging/logging.h"
#include "spark/md/GeneralMarketDataListener.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/md/MDProducer.h"
@@ -1,10 +1,10 @@
#include "MarginServiceCore.h"
#include "Position.h"
#include "logging/logging.h"
#include "spark/api/services/AuthService.pb.h"
#include "spark/config/config.h"
#include "spark/config/protos/span.pb.h"
#include "spark/logging/logging.h"
#include "spark/utils/Dispatcher.h"
#include "spark/utils/Uuid.h"
#include "spark/utils/bts_assert.h"
@@ -3,11 +3,11 @@
#include "Position.h"
#include "PositionAggregator.h"
#include "logging/logging.h"
#include "spark/api/service_helpers/AdminServiceUtils.h"
#include "spark/config/config.h"
#include "spark/config/protos/span.pb.h"
#include "spark/domain_model/Security.h"
#include "spark/logging/logging.h"
#include "spark/orderman/orderman_utils/orderman_utils.h"
#include "spark/os/os.h"
#include "spark/persistence/SqliteStore.h"
@@ -1,9 +1,8 @@
#include "PositionAggregator.h"
#include "Position.h"
#include "logging/logging.h"
#include "spark/domain_model/Security.h"
#include "spark/logging/logging.h"
#include "spark/md/MDClient.h"
#include "spark/orderman/orderman_utils/orderman_utils.h"
#include "spark/utils/Dispatcher.h"
@@ -1,11 +1,11 @@
#include "PositionMap.h"
#include "logging/logging.h"
#include "spark/api/service_helpers/PositionUtilities.h"
#include "spark/api/services/AdminService.client.pb.h"
#include "spark/api/services/AdminService.pb.h"
#include "spark/config/config.h"
#include "spark/config/protos/toggles.pb.h"
#include "spark/domain_model/Side.h"
#include "spark/logging/logging.h"
#include "spark/orderman/actors/SODPosition.h"
#include "spark/utils/PBUtils.h"
#include "spark/utils/bts_assert.h"
@@ -1,10 +1,10 @@
#include "PositionServicePNLHelper.h"
#include "logging/logging.h"
#include "spark/api/services/OrderService.client.pb.h"
#include "spark/config/config.h"
#include "spark/config/protos/cme_reference_data.pb.h"
#include "spark/config/protos/toggles.pb.h"
#include "spark/domain_model/Security.h"
#include "spark/logging/logging.h"
#include "spark/md/MDClient.h"
#include "spark/os/os.h"
#include "spark/utils/PBUtils.h"
@@ -1,6 +1,7 @@
#include "PositionServiceServer.h"
#include "PositionMap.h"
#include "PositionServicePNLHelper.h"
#include "logging/logging.h"
#include "spark/api/public/ServiceContext.h"
#include "spark/api/service_helpers/AdminServiceUtils.h"
#include "spark/api/service_helpers/PositionUtilities.h"
@@ -9,7 +10,6 @@
#include "spark/api/services/OrderService.client.pb.h"
#include "spark/api/services/OrderService.pb.h"
#include "spark/api/services/PositionService.pb.h"
#include "spark/logging/logging.h"
#include "spark/orderman/actors/SODPosition.h"
#include "spark/persistence/SqliteStore.h"
#include "spark/utils/PBUtils.h"
@@ -1,11 +1,11 @@
#include "BasePositionServiceServer.h"
#include "logging/logging.h"
#include "spark/api/public/ServiceContext.h"
#include "spark/api/services/AdminService.client.pb.h"
#include "spark/api/services/PositionService.client.pb.h"
#include "spark/config/config.h"
#include "spark/config/protos/orders_fills_cleanup.pb.h"
#include "spark/config/protos/position_cfg.pb.h"
#include "spark/logging/logging.h"
#include "spark/utils/PBUtils.h"
#include "spark/utils/bts_assert.h"
#include "spark/utils/datetime.h"
@@ -1,4 +1,5 @@
#include "MoneyFileServiceServer.h"
#include "logging/logging.h"
#include "spark/api/public/ServiceContext.h"
#include "spark/api/service_helpers/AdminServiceUtils.h"
#include "spark/api/services/AdminService.client.pb.h"
@@ -6,7 +7,6 @@
#include "spark/config/config.h"
#include "spark/config/protos/orders_fills_cleanup.pb.h"
#include "spark/config/protos/position_cfg.pb.h"
#include "spark/logging/logging.h"
#include "spark/utils/PBUtils.h"
#include "spark/utils/bts_assert.h"
#include "spark/utils/datetime.h"
@@ -1,7 +1,8 @@
#include "logging/logging.h"
#include "sparkd/position/import/parsing/money_parser.h"
#include "spark/api/service_helpers/AdminServiceUtils.h"
#include "spark/logging/logging.h"
#include "sparkd/position/import/parsing/parser_common.h"
#include <fmt/format.h>
#include <fstream>
@@ -1,7 +1,7 @@
#include "SubProcessRunner.h"
#include "logging/logging.h"
#include "spark/config/config.h"
#include "spark/config/protos/subprocess.pb.h"
#include "spark/logging/logging.h"
// #include <boost/filesystem.hpp>
#include <stdlib.h>
#include <sys/prctl.h>
@@ -1,7 +1,7 @@
#include "ErrorReportService.h"
#include "logging/logging.h"
#include "spark/config/config.h"
#include "spark/logging/logging.h"
#include "spark/utils/bts_assert.h"
#include "spark/utils/datetime.h"
#include "system/protobufs/system_config.pb.h"
@@ -1,7 +1,6 @@
#include "sparkd/user/AdminServiceDB.h"
#include "logging/logging.h"
#include "spark/api/services/AdminService.server.pb.h"
#include "spark/logging/logging.h"
#include "spark/persistence/SqliteStore.h"
#include "spark/utils/helper_macros.h"
@@ -1,7 +1,6 @@
#include "AdminServiceServer.h"
#include "logging/logging.h"
#include "spark/api/service_helpers/AdminServiceUtils.h"
#include "spark/logging/logging.h"
#include "spark/orderman/orderman_utils/orderman_utils.h"
#include "spark/persistence/SqliteStore.h"
#include "spark/utils/AuthToken.h"
@@ -1,9 +1,8 @@
#include "AuthServiceServer.h"
#include "logging/logging.h"
#include "spark/api/service_helpers/AdminServiceUtils.h"
#include "spark/api/services/AdminService.client.pb.h"
#include "spark/api/services/AuthService.pb.h"
#include "spark/logging/logging.h"
#include "spark/persistence/SqliteStore.h"
#include "spark/utils/AuthToken.h"
#include "spark/utils/PBUtils.h"
+62
View File
@@ -0,0 +1,62 @@
cmake_minimum_required(VERSION 3.23)
project(logging)
include(GNUInstallDirs)
include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
# Sources
file(
GLOB_RECURSE ${PROJECT_NAME}_SOURCES
RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
*.cpp
)
# Proto sources
file(
GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
*.proto
)
update_deps_file("${${PROJECT_NAME}_PROTO_SOURCES};")
# Generate proto cpp
message(STATUS "Generating proto cpp")
protobuf_generate_cpp(
CXX_SOURCES ${PROJECT_NAME}_CXX_PROTO_SOURCES
CXX_HEADERS ${PROJECT_NAME}_CXX_PROTO_HEADERS
CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
PROTO_INCLUDE_DIRS
${EXTERNALS_INCLUDE}
${CMAKE_CURRENT_SOURCE_DIR}
PROTO_SOURCES
${${PROJECT_NAME}_PROTO_SOURCES}
)
add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_CXX_PROTO_SOURCES} ${${PROJECT_NAME}_SOURCES})
target_include_directories(
${PROJECT_NAME} PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
)
target_link_directories(
${PROJECT_NAME} PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
$<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
)
target_link_libraries(${PROJECT_NAME} PUBLIC
system
protobuf
boost_system
boost_filesystem
)
foreach(_hdr ${${PROJECT_NAME}_CXX_PROTO_HEADERS})
file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
endforeach()
include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
install_basic()
+5
View File
@@ -0,0 +1,5 @@
@PACKAGE_INIT@
include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake")
check_required_components(@PROJECT_NAME@)
@@ -1,19 +1,8 @@
#include "spark/logging/LoggerHub.h"
#include "spark/logging/logging.h"
#include "spark/utils/bts_assert.h"
#include "system/lock_guard.h"
#include "logging/logger_hub.h"
#include "logging/logging.h"
#include <boost/circular_buffer.hpp>
#include <boost/container/flat_set.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
#include <ccapi_cpp/ccapi_logger.h>
#include <ccapi_cpp/service/ccapi_market_data_service_deribit.h>
#include <fmt/format.h>
#include <google/protobuf/io/gzip_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl.h>
#include <google/protobuf/text_format.h>
#include <mutex>
#include <string>
@@ -29,20 +18,23 @@ class CcapiLogger final : public Logger {
const std::string &line_num,
const std::string &msg) override {
auto level = to_bts_level_(severity);
if (!bts::log::_thread_logger.logger->is_loggable(bts::log::md, level)) {
if (!bts::logging::_thread_logger.logger->is_loggable(
bts::logging::md, level)) {
return;
}
// Add file name to cache to preserve it in memory until the log is
// [asynchronously] committed
const auto &cached_file_name = store_file_name_(file_name);
bts::log::_thread_logger.logger->enqueue(
bts::log::log_ts(), bts::log::md, level, cached_file_name.c_str(),
stoi(line_num), "{}", msg);
bts::logging::_thread_logger.logger->enqueue(
bts::logging::log_ts(), bts::logging::md, level,
cached_file_name.c_str(), stoi(line_num), "{}", msg);
}
static bts::log::Level to_bts_level_(const std::string &ccapi_severity) {
bts_assert(!ccapi_severity.empty());
static bts::logging::Level to_bts_level_(const std::string &ccapi_severity) {
if (ccapi_severity.empty()) {
return bts::Info;
}
// TRACE and DEBUG translate to Diag; all other levels have identical
// names in ccapi and spark
if (ccapi_severity[0] == 'T' || ccapi_severity[0] == 'D') {
@@ -60,7 +52,6 @@ class CcapiLogger final : public Logger {
if (ccapi_severity[0] == 'F') {
return bts::Fatal;
}
bts_assert(false);
return bts::Fatal;
}
@@ -78,7 +69,7 @@ class CcapiLogger final : public Logger {
// size would be limited, but not without cost in additional overhead for
// lookups.
const std::string &store_file_name_(const std::string &file_name) {
LOCK_GUARD(instance_mx_);
std::lock_guard<std::mutex> lock{instance_mx_};
return *file_name_store_.insert(file_name).first;
}
};
@@ -1,15 +1,12 @@
#pragma once
#include "spark/utils/windows_wrapper.h"
#include <array>
#include <limits>
#include <stdint.h>
#include <string>
#include <time.h>
namespace bts {
namespace log {
namespace bts::logging {
struct Level {
using Rep = uint8_t;
@@ -45,5 +42,4 @@ inline uint64_t log_ts() {
return result;
}
} // namespace log
} // namespace bts
} // namespace bts::logging
@@ -1,11 +1,10 @@
#pragma once
#include "Common.h"
#include "MsgPacking.h"
#include "spark/utils/warn_ctrl.h"
#include "logging/common.h"
#include "logging/msg_packing.h"
#include "system/warn_ctrl.h"
namespace bts {
namespace log {
namespace bts::logging {
using ArgUnpacker =
std::string (*)(ArgBuffer const &b, const char *msg_template);
@@ -22,7 +21,7 @@ struct LogRec {
ArgBuffer args;
template <typename... Args>
void pack_msg(const char *msg_t, Args &&... msg_args) {
void pack_msg(const char *msg_t, Args &&...msg_args) {
msg_template = msg_t;
unpacker = unpack_msg_args<
@@ -34,9 +33,4 @@ struct LogRec {
};
GCC_DIAG_RESTORE()
#ifndef _WIN32
// static_assert(sizeof(LogRec) == 256, "LogRec should be 128 bytes");
#endif
} // namespace log
} // namespace bts
} // namespace bts::logging
@@ -1,16 +1,16 @@
#pragma once
#include "LogRec.h"
#include "MsgPacking.h"
#include "spark/os/os.h"
#include "logging/log_rec.h"
#include "logging/msg_packing.h"
#include "system/os.h"
#include <assert.h>
#include <atomic>
#include <cstdint>
#include <numeric>
#include <thread>
namespace bts {
namespace log {
namespace bts::logging {
class Logger {
public:
@@ -39,13 +39,13 @@ class Logger {
template <typename... Args>
bool enqueue(
uint64_t ts,
std::uint64_t ts,
Subsystem subsystem,
Level lv,
const char *file,
int line,
const char *msg_template,
Args &&... msg_args) {
Args &&...msg_args) {
assert(_active);
lock();
if (_wcursor == _write_q->size())
@@ -65,19 +65,19 @@ class Logger {
size_t aggregate_hash() { return 0; }
template <typename Arg1, typename... Args>
size_t aggregate_hash(Arg1 &arg1, Args &&... args2) {
size_t aggregate_hash(Arg1 &arg1, Args &&...args2) {
return std::hash<Arg1>()(arg1) + aggregate_hash(args2...);
}
template <typename... Args>
bool enqueue_once_maybe(
uint64_t ts,
std::uint64_t ts,
Subsystem subsystem,
Level lv,
const char *file,
int line,
const char *msg_template,
Args &&... msg_args) {
Args &&...msg_args) {
const size_t filter_size = 1024 * 8; // 1kb
static std::vector<bool> s_bloom(filter_size);
auto arg_hash = aggregate_hash(msg_args...);
@@ -93,7 +93,8 @@ class Logger {
return true;
}
template <class S> void unqueue(S const &s) {
template <class S>
void unqueue(S const &s) {
size_t logged_msg_count = 0;
size_t dropped_msg_count = 0;
bool active = true;
@@ -121,8 +122,8 @@ class Logger {
unlock();
}
void
update_queue_limits(size_t initial_queue_size, size_t max_queue_growth) {
void update_queue_limits(
size_t initial_queue_size, size_t max_queue_growth) {
lock();
if (_wcursor == 0) {
// we can only do this if the queue is empty because each log
@@ -170,5 +171,5 @@ class Logger {
size_t _max_queue_size;
std::string _thread_name;
};
} // namespace log
} // namespace bts
} // namespace bts::logging
@@ -1,21 +1,19 @@
#include "LoggerHub.h"
#include "Logger.h"
#include "spark/os/os.h"
#include "spark/utils/bts_assert.h"
#include "spark/utils/bts_chrono.h"
#include "spark/utils/thread_manager.h"
#include "logger_hub.h"
#include "logging/logger.h"
#include "system/bts_chrono.h"
#include "system/os.h"
#include "system/protobufs/threads.pb.h"
#include "system/threading/thread_manager.h"
#include <algorithm>
#include <atomic>
#include <mutex>
#include <thread>
namespace bts {
namespace log {
namespace bts::logging {
using bts::system::protobufs::ThreadGroupCfg;
using bts::system::threading::ThreadManager;
std::atomic<size_t> thread_idx;
@@ -82,19 +80,24 @@ void LoggerHub::config(const LoggingConfig &cfg) {
}
void LoggerHub::start() {
std::lock_guard<std::mutex> lock{me->lock};
if (me->worker) {
// Already started
return;
}
std::cout << "Starting LoggerHub '" << me->logger_name << "' on thread "
<< os::get_thread_name() << ":" << std::this_thread::get_id()
<< std::endl;
bts_assert(!me->worker);
me->worker = std::make_unique<std::thread>(threading::make_thread(
me->logger_name.c_str(), std::bind(&LoggerHub::do_work, this),
me->thread_cfg));
me->worker = std::make_unique<std::thread>(ThreadManager::make_thread(
me->logger_name, std::bind(&LoggerHub::do_work, this), me->thread_cfg));
}
LoggerHub::~LoggerHub() {
std::lock_guard<std::mutex> lock{me->lock};
me->done = true;
if (me->worker)
if (me->worker) {
me->worker->join();
}
}
Logger &LoggerHub::make_logger() {
@@ -206,5 +209,5 @@ void LoggerHub::update_logger_queue_limits() {
l->update_queue_limits(
me->logger_initial_queue_size, me->logger_max_queue_growth);
}
} // namespace log
} // namespace bts
} // namespace bts::logging
@@ -1,13 +1,13 @@
#pragma once
#include "Common.h"
#include "LoggingConfig.h"
#include "Sink.h"
#include "logging/common.h"
#include "logging/logging_config.h"
#include "logging/sink.h"
#include <memory>
#include <string>
namespace bts {
namespace log {
namespace bts::logging {
class Logger;
struct LogRec;
@@ -33,5 +33,4 @@ class LoggerHub {
Filter get_base_filter() const;
};
} // namespace log
} // namespace bts
} // namespace bts::logging
@@ -1,16 +1,19 @@
#include "logging.h"
#include "LoggerHub.h"
#include "spark/config/config.h"
#include "spark/config/protos/deployment_type.pb.h"
#include "spark/config/protos/logging.pb.h"
#include "spark/os/os.h"
#include "spark/utils/Dispatcher.h"
#include "spark/utils/bts_assert.h"
#include "spark/utils/thread_manager.h"
#include "logging/logger_hub.h"
#include "logging/logging_config.h"
// #include "spark/config/config.h"
// #include "spark/config/protos/deployment_type.pb.h"
// #include "spark/config/protos/logging.pb.h"
#include "logging/protobufs/logging.h"
#include "logging/protobufs/logging.pb.h"
#include "system/facades/bts_assert.h"
#include "system/os.h"
#include "system/protobufs/system_config.pb.h"
#include "system/protobufs/threads.pb.h"
#include "system/threading/dispatcher.h"
#include "system/threading/thread_manager.h"
#ifdef SPARK_ENABLE_CCAPI
#include "spark/logging/ccapi_logger.h"
#include "logging/ccapi_logger.h"
#endif
#include <boost/circular_buffer.hpp>
@@ -25,10 +28,12 @@
#include <google/protobuf/text_format.h>
#include <mutex>
#include <string>
#include <utility>
namespace bts {
namespace log {
namespace bts::logging {
using bts::logging::protobufs::Logging;
using bts::logging::protobufs::SinkConfig;
using bts::system::protobufs::SystemConfig;
using bts::system::protobufs::ThreadGroupCfg;
using Path = boost::filesystem::path;
@@ -46,13 +51,14 @@ typename M::const_iterator find_key(
}
Path log_path() {
return Path{config::get<SystemConfig>().data_path()} /
fmt::format("logging.{}/", os::get_application_name());
return Path{"/home/bts/.local/share/bts/sparkd/data/logging.sparkd"};
// return Path{config::get<SystemConfig>().data_path()} /
// fmt::format("logging.{}/", os::get_application_name());
}
void start_default_logging_thread() { _default_logger_hub.start(); }
Filter get_filter(LoggingConfig &cfg, config::SinkConfig const &scfg) {
Filter get_filter(LoggingConfig &cfg, SinkConfig const &scfg) {
Filter result;
for (auto &l : result)
l = Info;
@@ -84,16 +90,15 @@ Filter get_filter(LoggingConfig &cfg, config::SinkConfig const &scfg) {
return result;
}
void configure_logging(LoggingConfig const &cfg) {
void configure_logging(const LoggingConfig &cfg) {
_default_logger_hub.config(cfg);
}
void configure_logging(bts::config::Logging const &cfg) {
void configure_logging(const Logging &cfg) {
configure_logging(cfg, ThreadGroupCfg());
}
void configure_logging(
bts::config::Logging const &cfg, const ThreadGroupCfg &thread_cfg) {
void configure_logging(const Logging &cfg, const ThreadGroupCfg &thread_cfg) {
LoggingConfig real_cfg;
real_cfg.deployment_type = config::get<config::DeploymentType>().type();
// real_cfg.deployment_type = config::get<config::DeploymentType>().type();
real_cfg.logger_initial_queue_size = cfg.initial_queue_size();
real_cfg.logger_max_queue_growth = cfg.max_queue_growth();
@@ -102,18 +107,18 @@ void configure_logging(
{Sys, "Sys"}};
real_cfg.subsystem_names = {
{log::Gen, "Gen"}, {log::Cfg, "Cfg"},
{log::QBus, "QBus"}, {log::Log, "Log"},
{log::Jetfix, "Jetfix"}, {log::md, "md"},
{log::Api, "Api"}, {log::persist, "persist"},
{log::User, "User"}, {log::OrderMan, "OrderMan"},
{log::Assert, "Assert"}, {log::Engine, "Engine"},
{log::Ui, "UI"}, {log::ExSim, "ExSim"},
{log::Mercury, "Mercury"}, {log::Testing, "Testing"},
{log::Latency, "Latency"}, {log::DataFlow, "DataFlow"},
{log::Plugin, "Plugin"}, {log::Sdk, "Sdk"},
{log::Http, "Http"}, {log::Spartan, "Spartan"},
{log::iLink3, "iLink3"},
{Gen, "Gen"}, {Cfg, "Cfg"},
{QBus, "QBus"}, {Log, "Log"},
{Jetfix, "Jetfix"}, {md, "md"},
{Api, "Api"}, {persist, "persist"},
{User, "User"}, {OrderMan, "OrderMan"},
{Assert, "Assert"}, {Engine, "Engine"},
{Ui, "UI"}, {ExSim, "ExSim"},
{Mercury, "Mercury"}, {Testing, "Testing"},
{Latency, "Latency"}, {DataFlow, "DataFlow"},
{Plugin, "Plugin"}, {Sdk, "Sdk"},
{Http, "Http"}, {Spartan, "Spartan"},
{iLink3, "iLink3"},
};
real_cfg.sinks["console"] = std::make_pair(
@@ -129,7 +134,7 @@ void configure_logging(
Path log_cache_file_path() { return log_path() / "log_control"; }
LoggingConfigPoller::LoggingConfigPoller(
bts::config::Logging const &cfg, const ThreadGroupCfg &thread_cfg)
const Logging &cfg, const ThreadGroupCfg &thread_cfg)
: _poller(
utils::Dispatcher::background(), 1000, [=, this]() { poll_once(); }),
_last_ts(std::time(nullptr)), _log_cache_file(log_cache_file_path()),
@@ -139,11 +144,13 @@ LoggingConfigPoller::LoggingConfigPoller(
LoggingConfigPoller::LoggingConfigPoller()
: LoggingConfigPoller(
config::get<config::Logging>(), threading::get_thread_cfg("logger")) {
}
logging::protobufs::default_logging_config(),
system::threading::ThreadManager{
system::threading::ThreadManager::default_thread_config()}
.get_thread_cfg("logger")) {}
void LoggingConfigPoller::init(
bts::config::Logging const &cfg, const ThreadGroupCfg &thread_cfg) {
Logging const &cfg, const ThreadGroupCfg &thread_cfg) {
auto parent = _log_cache_file.parent_path();
if (!boost::filesystem::is_directory(parent))
bts_assert(boost::filesystem::create_directories(parent));
@@ -164,7 +171,7 @@ void LoggingConfigPoller::poll_once() {
if (current_ts != _last_ts) {
_last_ts = current_ts;
bts::config::Logging cfg;
Logging cfg;
std::ifstream fs(_log_cache_file.string());
google::protobuf::io::IstreamInputStream gs(&fs);
if (google::protobuf::TextFormat::Parse(&gs, &cfg)) {
@@ -181,7 +188,9 @@ LoggingConfigPoller::~LoggingConfigPoller() {
_poller.stop();
}
namespace interop {
} // namespace bts::logging
namespace bts::logging::interop {
const char *intern(const char *str) {
static std::mutex mutex;
@@ -203,17 +212,15 @@ const char *intern(const char *str) {
extern "C" BTS_EXPORT void Logger_log(
int level, int subsystem, const char *message, const char *file, int line) {
if (bts::log::_thread_logger.logger != nullptr) {
bts::log::_thread_logger.logger->enqueue(
log::log_ts(), log::Subsystem{(uint8_t)subsystem},
log::Level{(uint8_t)level}, intern(file), line, "{}",
if (bts::logging::_thread_logger.logger != nullptr) {
bts::logging::_thread_logger.logger->enqueue(
logging::log_ts(), logging::Subsystem{(uint8_t)subsystem},
logging::Level{(uint8_t)level}, intern(file), line, "{}",
std::string(message));
}
}
} // namespace interop
} // namespace log
} // namespace bts
} // namespace bts::logging::interop
#ifdef SPARK_ENABLE_CCAPI
namespace ccapi {
+129
View File
@@ -0,0 +1,129 @@
#pragma once
#include "logging/common.h"
#include "logging/logger.h"
#include "logging/logger_hub.h"
#include "logging/protobufs/logging.pb.h"
#include "system/bts_chrono.h"
#include "system/threading/dispatcher.h"
#include <atomic>
#include <boost/filesystem/path.hpp>
#include <thread>
namespace bts::system::protobufs {
class ThreadGroupCfg;
}
namespace bts::config {
class Logging;
} // namespace bts::config
namespace bts {
// Please keep this list in sync with Log.cs
constexpr bts::logging::Level NoLog{0};
constexpr bts::logging::Level Diag{NoLog.ordinal + 1};
constexpr bts::logging::Level Info{Diag.ordinal + 1};
constexpr bts::logging::Level Warn{Info.ordinal + 1};
constexpr bts::logging::Level Error{Warn.ordinal + 1};
constexpr bts::logging::Level Fatal{Error.ordinal + 1};
constexpr bts::logging::Level Sys{logging::Level::Max - 1};
} // namespace bts
namespace bts::logging {
// Please keep this list in sync with Log.cs
constexpr logging::Subsystem Gen{0};
constexpr logging::Subsystem Cfg{Gen.ordinal + 1};
constexpr logging::Subsystem QBus{Cfg.ordinal + 1};
constexpr logging::Subsystem Log{QBus.ordinal + 1};
constexpr logging::Subsystem Jetfix{Log.ordinal + 1};
constexpr logging::Subsystem md{Jetfix.ordinal + 1};
constexpr logging::Subsystem Api{md.ordinal + 1};
constexpr logging::Subsystem persist{Api.ordinal + 1};
constexpr logging::Subsystem User{persist.ordinal + 1};
constexpr logging::Subsystem OrderMan{User.ordinal + 1};
constexpr logging::Subsystem Assert{OrderMan.ordinal + 1};
constexpr logging::Subsystem Engine{Assert.ordinal + 1};
constexpr logging::Subsystem Ui{Engine.ordinal + 1};
constexpr logging::Subsystem ExSim{Ui.ordinal + 1};
constexpr logging::Subsystem Mercury{ExSim.ordinal + 1};
constexpr logging::Subsystem Testing{Mercury.ordinal + 1};
constexpr logging::Subsystem Latency{Testing.ordinal + 1};
constexpr logging::Subsystem DataFlow{Latency.ordinal + 1};
constexpr logging::Subsystem Plugin{DataFlow.ordinal + 1};
constexpr logging::Subsystem Sdk{Plugin.ordinal + 1};
constexpr logging::Subsystem Http{Sdk.ordinal + 1};
constexpr logging::Subsystem Spartan{Http.ordinal + 1};
constexpr logging::Subsystem iLink3{Spartan.ordinal + 1};
} // namespace bts::logging
namespace bts::logging {
struct LoggerHandle {
LoggerHandle(LoggerHandle const &) = delete;
LoggerHandle &operator=(LoggerHandle const &) = delete;
LoggerHandle() : logger(nullptr) {}
LoggerHandle(Logger &l) : logger(&l) {}
~LoggerHandle() {
if (logger)
logger->mark_inactive();
}
Logger *logger;
};
struct CharPtrWrapper {
const char *ptr;
};
inline std::ostream &operator<<(std::ostream &s, CharPtrWrapper const &ptr) {
s << ptr.ptr;
return s;
}
extern thread_local LoggerHandle _thread_logger;
void configure_logging(LoggingConfig const &cfg);
void configure_logging(
const logging::protobufs::Logging &cfg,
const system::protobufs::ThreadGroupCfg &thread_cfg);
void configure_logging(const bts::logging::protobufs::Logging &cfg);
void start_default_logging_thread();
class LoggingConfigPoller {
public:
LoggingConfigPoller();
LoggingConfigPoller(
const logging::protobufs::Logging &cfg,
const system::protobufs::ThreadGroupCfg &thread_cfg);
~LoggingConfigPoller();
private:
void init(
const logging::protobufs::Logging &cfg,
const system::protobufs::ThreadGroupCfg &thread_cfg);
void poll_once();
utils::DispatcherTimer _poller;
std::time_t _last_ts;
boost::filesystem::path _log_cache_file;
std::atomic<bool> _done;
};
boost::filesystem::path log_path();
#define LOG(subsystem, level, ...) \
bts::logging::_thread_logger.logger->is_loggable( \
bts::logging::subsystem, level) && \
bts::logging::_thread_logger.logger->enqueue( \
bts::logging::log_ts(), bts::logging::subsystem, level, __FILE__, \
__LINE__, __VA_ARGS__)
#define LOG_ONCE_MAYBE(subsystem, level, ...) \
bts::logging::_thread_logger.logger->is_loggable( \
bts::logging::subsystem, level) && \
bts::logging::_thread_logger.logger->enqueue_once_maybe( \
bts::logging::log_ts(), logging::subsystem, level, __FILE__, \
__LINE__, __VA_ARGS__)
} // namespace bts::logging
@@ -1,14 +1,13 @@
#pragma once
#include "Common.h"
#include "Sink.h"
#include "logging/common.h"
#include "logging/sink.h"
#include "system/protobufs/threads.pb.h"
#include <map>
#include <vector>
namespace bts {
namespace log {
namespace bts::logging {
struct LoggingConfig {
LoggingConfig()
@@ -25,5 +24,4 @@ struct LoggingConfig {
system::protobufs::ThreadGroupCfg thread_cfg;
};
} // namespace log
} // namespace bts
} // namespace bts::logging
@@ -2,12 +2,10 @@
#include <array>
#include <fmt/ostream.h>
#include <iostream>
#include <type_traits>
#include <vector>
namespace bts {
namespace log {
namespace bts::logging {
template <class Tuple, std::size_t... Is>
std::string format_tuple(
@@ -45,7 +43,10 @@ std::string unpack_msg_args(
return result;
}
template <typename... Args> struct CheckPtr { const static auto value = true; };
template <typename... Args>
struct CheckPtr {
const static auto value = true;
};
template <typename Head, typename... Tail>
struct CheckPtr<Head, Tail...> : public CheckPtr<Tail...> {
@@ -53,7 +54,7 @@ struct CheckPtr<Head, Tail...> : public CheckPtr<Tail...> {
};
template <size_t SZ, typename... Args>
void pack_msg_args(std::array<uint8_t, SZ> &buffer, Args &&... args) {
void pack_msg_args(std::array<uint8_t, SZ> &buffer, Args &&...args) {
using Store = std::tuple<typename std::remove_reference<Args>::type...>;
static_assert(
CheckPtr<typename std::remove_reference<Args>::type...>::value,
@@ -66,5 +67,4 @@ void pack_msg_args(std::array<uint8_t, SZ> &buffer, Args &&... args) {
}
}
} // namespace log
} // namespace bts
} // namespace bts::logging
@@ -0,0 +1,22 @@
#pragma once
#include "logging/protobufs/logging.pb.h"
namespace bts::logging::protobufs {
inline Logging default_logging_config() {
Logging rv;
*rv.mutable_console()->mutable_default_level() = "Info";
*rv.mutable_syslog()->mutable_default_level() = "Info";
*rv.mutable_file()->mutable_default_level() = "Info";
#ifdef _WIN32
rv.set_initial_queue_size(1024);
rv.set_max_queue_growth(100);
#else
rv.set_initial_queue_size(120001);
rv.set_max_queue_growth(4);
#endif
return rv;
}
} // namespace bts::logging::protobufs
@@ -1,8 +1,8 @@
syntax = "proto3";
package bts.config;
package bts.logging.protobufs;
option csharp_namespace = "bts.config";
option csharp_namespace = "bts.logging.protobufs";
message SubsystemConfig {
string name = 1;
@@ -1,24 +1,27 @@
#include "Sink.h"
#include "sink.h"
#include "spark/logging/logging.h"
#include "spark/os/os.h"
#include "spark/utils/bts_assert.h"
#include "spark/utils/datetime.h"
#include "spark/utils/file_utils.h"
#include "logging/logging.h"
#include "system/facades/bts_assert.h"
#include "system/file_utils.h"
#include "system/os.h"
#include "system/time_stamp.h"
#include <boost/date_time/gregorian/gregorian.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/path.hpp>
#include <fmt/format.h>
#include <fmt/ostream.h>
#include <fstream>
#include <iostream>
#include <sstream>
#include <time.h>
#ifndef _WIN32
#include <syslog.h>
#endif
namespace bts {
namespace log {
namespace bts::logging {
struct InitSysLog {
InitSysLog() {
@@ -30,7 +33,8 @@ struct InitSysLog {
auto formatter(LoggingConfig const &cfg) {
return [cfg](LogMsg const &m) {
auto iso_date = datetime::iso_date_time_string(datetime::local_now());
auto now_us = boost::posix_time::microsec_clock::local_time();
auto iso_date = boost::posix_time::to_iso_extended_string(now_us);
iso_date[sizeof("YYYY-MM-DD") - 1] = ' ';
#ifdef _WIN32
return fmt::format(
@@ -57,7 +61,8 @@ auto formatter(LoggingConfig const &cfg) {
Sink console_sink(LoggingConfig const &) {
return [](LogMsg const &m) {
auto iso_date = datetime::iso_date_time_string(datetime::local_now());
auto now_us = boost::posix_time::microsec_clock::local_time();
auto iso_date = boost::posix_time::to_iso_extended_string(now_us);
iso_date[sizeof("YYYY-MM-DD") - 1] = ' ';
std::cout << fmt::format(
"{}|{:<8}|{:<5}|{:<35}| {}", iso_date, m.subsystem_name,
@@ -134,5 +139,4 @@ Sink null_sink() {
return [](const LogMsg &) {};
}
} // namespace log
} // namespace bts
} // namespace bts::logging
@@ -1,14 +1,14 @@
#pragma once
#include "Common.h"
#include "spark/utils/warn_ctrl.h"
#include "logging/common.h"
#include "system/warn_ctrl.h"
#include <functional>
#include <string>
#include <thread>
namespace bts {
namespace log {
namespace bts::logging {
struct LoggingConfig;
GCC_DIAG_OFF("-Weffc++")
struct LogMsg {
@@ -33,5 +33,5 @@ Sink console_sink(LoggingConfig const &cfg);
Sink syslog_sink(LoggingConfig const &cfg);
Sink file_sink(LoggingConfig const &cfg);
Sink null_sink();
} // namespace log
} // namespace bts
} // namespace bts::logging
@@ -0,0 +1,82 @@
// #include "catch/catch.hpp"
// // #include "spark/config/config.h"
// #include "logging/logging.h"
// #include "logging/protobufs/logging.h"
// #include "logging/protobufs/logging.pb.h"
// #include "system/bts_chrono.h"
// #include <fmt/chrono.h>
// #include <fmt/format.h>
// namespace bts {
// int logging_tests;
// TEST_CASE("Logging Tests", "[logging] [benchmark]") {
// auto sz = 1000000;
// log::Filter f;
// for (auto &l : f)
// l = Diag;
// log::LoggingConfig cfg;
// cfg.logger_initial_queue_size = sz * 10;
// cfg.sinks["null_sink"] = std::make_pair([](log::LogMsg const &) {}, f);
// log::configure_logging(cfg);
// LOG(Gen, Info, "Kicking things off");
// SECTION("timestamp_interpolation") {
// auto start = bts::Timer::now();
// for (int i = 0; i < sz; i++)
// LOG(Gen, Info, "Hello World. It is now: {}",
// Timer::now().time_since_epoch());
// auto stop = Timer::now();
// LOG(Testing, Info, "Logging: ns/msg for log with timestamps: {} tt: {} ",
// bts::ns_diff(stop, start) / sz, bts::ns_diff(stop, start));
// }
// SECTION("int_interpolation") {
// auto start = bts::Timer::now();
// for (int i = 0; i < sz; i++)
// LOG(Gen, Info, "Hello World. This is interpolating a number {0}", 12);
// auto stop = bts::Timer::now();
// LOG(Testing, Info,
// "Logging: ns/msg for log with simple number interpolation: {} tt: "
// "{} ",
// bts::ns_diff(stop, start) / sz, bts::ns_diff(stop, start));
// }
// SECTION("large_string_interpolation") {
// std::array<std::string, 8> strs;
// int i = 0;
// strs[i++] = "aaaaaaaaaaaaaaaaa";
// strs[i++] = "aaaaaaaaaaaaaaaaa";
// strs[i++] = "aaaaaaaaaaaaaaaaa";
// strs[i++] = "aaaaaaaaaaaaaaaaa";
// strs[i++] = "aaaaaaaaaaaaaaaaa";
// strs[i++] = "aaaaaaaaaaaaaaaaa";
// strs[i++] = "aaaaaaaaaaaaaaaaa";
// strs[i++] = "aaaaaaaaaaaaaaaaa";
// auto start = bts::Timer::now();
// for (int i = 0; i < sz; i++)
// LOG(Gen, Info, "Hello World. This is interpolating a string ",
// strs[i & 0x00000007]);
// auto stop = bts::Timer::now();
// LOG(Testing, Info,
// "Logging: ns/msg for log with large string interpolation: {} tt: {} ",
// bts::ns_diff(stop, start) / sz, bts::ns_diff(stop, start));
// }
// SECTION("log_skipping") {
// auto start = bts::Timer::now();
// for (int i = 0; i < sz; i++)
// LOG(Gen, Diag, "Hello World. I am {} years old!", i);
// auto stop = bts::Timer::now();
// LOG(Testing, Info, "Logging: ns/msg for log skipping: {} tt: {} ",
// bts::ns_diff(stop, start) / sz, bts::ns_diff(stop, start));
// }
// auto cfg2 = logging::protobufs::default_logging_config();
// cfg2.mutable_console()->set_default_level("Fatal");
// log::configure_logging(cfg2);
// }
// } // namespace bts
@@ -3,6 +3,7 @@ function(get_link_libraries dest)
system
domain
protobuf
logging
double-conversion
boost_system
boost_thread
@@ -1,6 +1,6 @@
#include "ApiClientInterop.h"
#include "logging/logging.h"
#include "spark/config/config.h"
#include "spark/logging/logging.h"
#include "spark/quickbus/quickbus.h"
#include "spark/utils/Dispatcher.h"
#include "spark/utils/bts_assert.h"
@@ -1,5 +1,5 @@
#include "ApiServerInterop.h"
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include "spark/quickbus/quickbus.h"
#include "spark/utils/bts_assert.h"
@@ -1,8 +1,8 @@
#if !defined(_WIN32) && defined(SPARK_ENABLE_CCAPI)
#include "MarketDataConfigUtils.h"
#include "logging/logging.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/domain_model/Security.pb.h"
#include "spark/logging/logging.h"
//#if !defined(_WIN32) && defined(SPARK_ENABLE_CCAPI)
namespace bts::md {
@@ -74,4 +74,4 @@ MarketDataConfigUtils::get_effective_recording_config(
}
} // namespace bts::md
#endif
#endif
@@ -5,7 +5,7 @@
#include "RawUpdate.h"
#include "Result.h"
#include "ServiceContext.h"
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include <functional>
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
@@ -2,8 +2,9 @@
#include "ServiceContext.h"
#include "ApiServer.h"
#include "QuickbusClientChannel.h"
#include "logging/logging.h"
#include "spark/api/services/AuthService.client.pb.h"
#include "spark/logging/logging.h"
#include <memory>
namespace bts {
@@ -1,8 +1,8 @@
#include "PositionUtilities.h"
#include "logging/logging.h"
#include "spark/api/services/OrderService.client.pb.h"
#include "spark/api/services/OrderService.pb.h"
#include "spark/api/services/PositionService.server.pb.h"
#include "spark/logging/logging.h"
namespace bts {
namespace position {
@@ -150,4 +150,4 @@ double PositionUtilities::compute_pnl_entry(
}
} // namespace position
} // namespace bts
} // namespace bts
@@ -1,9 +1,9 @@
#pragma once
#include "logging/logging.h"
#include "spark/api/services/OrderService.client.pb.h"
#include "spark/api/services/OrderService.pb.h"
#include "spark/api/services/PositionService.server.pb.h"
#include "spark/logging/logging.h"
namespace bts {
namespace position {
@@ -56,4 +56,4 @@ class PositionUtilities {
};
} // namespace position
} // namespace bts
} // namespace bts
@@ -1,7 +1,7 @@
#pragma once
#include "logging/logging.h"
#include "spark/api/services/OrderService.pb.h"
#include "spark/logging/logging.h"
#include "system/FMT_IMPL.h"
#include "system/enum_macros.h"
#include "system/json/json_reader.h"
+22 -21
View File
@@ -1,6 +1,6 @@
#pragma once
#include "spark/logging/logging.h"
#include "logging/logging.h"
#include "spark/utils/bts_no_copy.h"
#include <memory>
@@ -23,17 +23,17 @@ class ConfigLog {
void warning(const std::string &msg);
bool enqueue(
bts::log::Level level, const char *file, int line, std::string msg);
bts::logging::Level level, const char *file, int line, std::string msg);
template <typename... Args>
bool enqueue(
bts::log::Level level,
bts::logging::Level level,
const char *file,
int line,
const char *msg_template,
Args &&...msg_args);
bool is_loggable(bts::log::Level level) const;
bool is_loggable(bts::logging::Level level) const;
bool print_errors() const;
void set_print_errors(bool value);
@@ -42,8 +42,8 @@ class ConfigLog {
void start();
bool print_errors_;
log::LoggerHub logger_hub_;
log::Logger &logger_;
logging::LoggerHub logger_hub_;
logging::Logger &logger_;
};
//////////////////////////////////////////////////
@@ -53,23 +53,23 @@ class ConfigLog {
inline ConfigLog::ConfigLog()
: print_errors_(true), logger_hub_("config_logger"),
logger_(logger_hub_.make_logger()) {
log::LoggingConfig cfg;
logging::LoggingConfig cfg;
cfg.level_names = {{Diag, "Diag"}, {Info, "Info"}, {Warn, "Warn"},
{Error, "Error"}, {Fatal, "Fatal"}, {Sys, "Sys"}};
cfg.subsystem_names = {{log::Gen, "Gen"}, {log::Log, "Log"}};
cfg.subsystem_names = {{logging::Gen, "Gen"}, {logging::Log, "Log"}};
cfg.logger_initial_queue_size = 10;
log::Filter f;
logging::Filter f;
for (auto &l : f) {
l = Warn;
}
cfg.deployment_type = "Config";
f[log::Cfg.ordinal] = Warn;
cfg.sinks["console"] = std::make_pair(log::console_sink(cfg), f);
f[log::Cfg.ordinal] = Info;
cfg.sinks["syslog"] = std::make_pair(log::syslog_sink(cfg), f);
f[logging::Cfg.ordinal] = Warn;
cfg.sinks["console"] = std::make_pair(logging::console_sink(cfg), f);
f[logging::Cfg.ordinal] = Info;
cfg.sinks["syslog"] = std::make_pair(logging::syslog_sink(cfg), f);
logger_hub_.config(cfg);
start();
@@ -87,7 +87,7 @@ inline void ConfigLog::info(const std::string &msg) {
return;
}
logger_.enqueue(
log::log_ts(), log::Cfg, Info, __FILE__, __LINE__, "{0}", msg);
logging::log_ts(), logging::Cfg, Info, __FILE__, __LINE__, "{0}", msg);
}
inline void ConfigLog::error(const std::string &msg) {
@@ -95,7 +95,7 @@ inline void ConfigLog::error(const std::string &msg) {
return;
}
logger_.enqueue(
log::log_ts(), log::Cfg, Error, __FILE__, __LINE__, "{0}", msg);
logging::log_ts(), logging::Cfg, Error, __FILE__, __LINE__, "{0}", msg);
}
inline void ConfigLog::warning(const std::string &msg) {
@@ -103,21 +103,21 @@ inline void ConfigLog::warning(const std::string &msg) {
return;
}
logger_.enqueue(
log::log_ts(), log::Cfg, Warn, __FILE__, __LINE__, "{0}", msg);
logging::log_ts(), logging::Cfg, Warn, __FILE__, __LINE__, "{0}", msg);
}
inline bool ConfigLog::enqueue(
bts::log::Level level, const char *file, int line, std::string msg) {
bts::logging::Level level, const char *file, int line, std::string msg) {
return enqueue(level, file, line, "{}", std::move(msg));
}
inline bool ConfigLog::is_loggable(bts::log::Level level) const {
return print_errors_ && logger_.is_loggable(bts::log::Cfg, level);
inline bool ConfigLog::is_loggable(bts::logging::Level level) const {
return print_errors_ && logger_.is_loggable(bts::logging::Cfg, level);
}
template <typename... Args>
inline bool ConfigLog::enqueue(
bts::log::Level level,
bts::logging::Level level,
const char *file,
int line,
const char *msg_template,
@@ -126,7 +126,8 @@ inline bool ConfigLog::enqueue(
return false;
}
return logger_.enqueue(
log::log_ts(), log::Cfg, level, file, line, msg_template, msg_args...);
logging::log_ts(), logging::Cfg, level, file, line, msg_template,
msg_args...);
}
} // namespace internal
+1 -1
View File
@@ -1,7 +1,7 @@
#pragma once
#include "logging/logging.h"
#include "spark/domain_model/Security.pb.h"
#include "spark/logging/logging.h"
#include "system/FMT_IMPL.h"
#include "system/enum_macros.h"
#include "system/json/json_reader.h"
+1 -1
View File
@@ -1,7 +1,7 @@
#pragma once
#include "logging/logging.h"
#include "spark/domain_model/Security.pb.h"
#include "spark/logging/logging.h"
#include "system/FMT_IMPL.h"
#include "system/enum_macros.h"
#include "system/json/json_reader.h"
Loaded 100 of 235 files, more files were not shown because too many files have changed in this diff. Show more