diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs b/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs index 6d8b90596..ebd7ce860 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs @@ -25,7 +25,7 @@ static class Program { cfg.MergeWithConfigFile(); cfg.MergeWith(DataStreamQueryAnalyzerConfig.Default()); PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub)); - Configuration.initialize(cfg.Sparkd); + await Configuration.InitializeAsync(cfg.Sparkd); Log.Gen.info("Starting DataStreamQueryAnalyzer"); var mainForm = new MainForm(); if (cfg.StreamFile is not null) { diff --git a/src/client/spark_api/Services/SparkApiServices.cs b/src/client/spark_api/Services/SparkApiServices.cs index e7fb10e94..30b413e47 100644 --- a/src/client/spark_api/Services/SparkApiServices.cs +++ b/src/client/spark_api/Services/SparkApiServices.cs @@ -31,8 +31,8 @@ public static class SparkApiServices { Console.WriteLine("SparkApi services have been disabled"); return services; } - if (!services.Any(s => s.ServiceType == typeof(SparkService))) { - /// initializes configuration and logging (this should be done + if (!services.Any(s => s.ServiceType == typeof(ISparkConfigLoader))) { + /// initializes configuration and logging (this should be done /// before any api-related services run). throw new InvalidOperationException("Must add SparkService first"); } diff --git a/src/client/spark_api/api/ManagedQuickbusWrapper.cs b/src/client/spark_api/api/ManagedQuickbusWrapper.cs new file mode 100644 index 000000000..0bdadf6f1 --- /dev/null +++ b/src/client/spark_api/api/ManagedQuickbusWrapper.cs @@ -0,0 +1,35 @@ +#nullable enable +namespace Spark.Core; + +using bts.api.core; + +using System; + +sealed class ManagedQuickbusWrapper : IDisposable { + readonly object lockObj = new(); + IntPtr handle; + + public ManagedQuickbusWrapper() + : this(ApiNativeMethods.api_create_quickbus(null, 0)) {} + + internal ManagedQuickbusWrapper(IntPtr handle) => + this.handle = handle; + + public static ManagedQuickbusWrapper None => new(IntPtr.Zero); + + public IntPtr Handle => handle; + + public bool IsDisposed => handle == IntPtr.Zero; + + public void Dispose() { + if (IsDisposed) { + return; + } + lock (lockObj) { + if (handle != IntPtr.Zero) { + ApiNativeMethods.api_destroy_quickbus(handle); + handle = IntPtr.Zero; + } + } + } +} diff --git a/src/client/spark_api/api/ServiceContext.cs b/src/client/spark_api/api/ServiceContext.cs index 7a7489f3b..e8d0290ee 100644 --- a/src/client/spark_api/api/ServiceContext.cs +++ b/src/client/spark_api/api/ServiceContext.cs @@ -2,36 +2,53 @@ namespace bts.api.core { using bts.api.user; - using Microsoft.Extensions.Options; - using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using System.Reflection; + using Spark.Core.Services; + using Spark.Core; + using System.Threading; public sealed class ServiceContext : Spark.Api.IServiceProvider, IDisposable { - IntPtr _qbus; + readonly object lockObj = new(); + LazyOnce? quickbusBacking; + bool isDisposed; + AuthToken? _token; - public ServiceContext() { - _qbus = ApiNativeMethods.api_create_quickbus(null, 0); + public ServiceContext(ISparkConfigLoader configLoader) { + quickbusBacking = new(QuickbusFactory, lockObj); + + ManagedQuickbusWrapper QuickbusFactory() { + if (isDisposed) { + return ManagedQuickbusWrapper.None; + } + configLoader.InitializeAsync(CancellationToken.None).Wait(); + return new(); + } } - internal ServiceContext(IntPtr qbus) => - _qbus = qbus; + internal ServiceContext(IntPtr qbus) { + quickbusBacking = new(QuickbusFactory, lockObj); + + ManagedQuickbusWrapper QuickbusFactory() { + // This code is executed in a lock (since we passed 'lockObj' in to the constructor), so + // no need to worry about concurrency when reading 'isDisposed'. + Log.assert(!isDisposed); + if (isDisposed) { + + } + return new(qbus); + } + } public AuthTokenImmutable? token => _token?.immutable(); public UserTuple? user { get; private set; } public bool is_authenticated => _token is not null; - internal IntPtr native_context => _qbus; - - public static ServiceContext create(string? config_server = null) { - BTS.init(config_server); - var qbus = ApiNativeMethods.api_create_quickbus(null, 0); - return new ServiceContext(qbus); - } + internal IntPtr native_context => quickbusBacking?.Value?.Handle ?? throw new ObjectDisposedException(nameof(ServiceContext)); public T locate(string qualifier = "") where T : class, IApiClient { var ctr = typeof(T).GetConstructors(BindingFlags.NonPublic | BindingFlags.Instance) @@ -87,9 +104,25 @@ namespace bts.api.core { } public void Dispose() { - if (_qbus != IntPtr.Zero) { - ApiNativeMethods.api_destroy_quickbus(_qbus); - _qbus = IntPtr.Zero; + if (isDisposed) { + return; + } + ManagedQuickbusWrapper? oldQuickbus; + lock (lockObj) { + if (isDisposed) { + return; + } + isDisposed = true; + if (quickbusBacking?.IsValueCreated ?? false) { + // IMPORTANT: Don't access quickbusBacking.Value unless you are SURE that the value + // has already been created (otherwise the factory function will deadlock, since it + // also locks 'lockObj'). + oldQuickbus = quickbusBacking.Value; + quickbusBacking.Value.Dispose(); + } + // Set quickbusBacking to null to prevent the quickbus from being created after dispose + // has been called (if it hasn't been created already). + quickbusBacking = null; } } diff --git a/src/client/spark_core/LazyOnce.cs b/src/client/spark_core/LazyOnce.cs index 4babc3ab3..498fc5b9e 100644 --- a/src/client/spark_core/LazyOnce.cs +++ b/src/client/spark_core/LazyOnce.cs @@ -8,10 +8,17 @@ namespace Spark.Core { /// public class LazyOnce { readonly Func _create; + readonly object syncRoot; Optional _value; - public LazyOnce(Func create) => + public LazyOnce(Func create) + : this(create, new()) { } + + public LazyOnce(Func create, object syncRoot) { _create = create; + ArgumentNullException.ThrowIfNull(syncRoot); + this.syncRoot = syncRoot; + } public bool IsValueCreated => _value.has_value; @@ -19,7 +26,7 @@ namespace Spark.Core { get { // check has_value outside of lock since, once set to true, it doesn't change if (!_value.has_value) { - lock (_create) { + lock (syncRoot) { // check has_value again (may have become true since first check) if (!_value.has_value) { _value = _create(); diff --git a/src/client/spark_core/Services/ISparkConfigLoader.cs b/src/client/spark_core/Services/ISparkConfigLoader.cs new file mode 100644 index 000000000..651176f19 --- /dev/null +++ b/src/client/spark_core/Services/ISparkConfigLoader.cs @@ -0,0 +1,12 @@ +#nullable enable +namespace Spark.Core.Services; + +using System.Threading; +using System.Threading.Tasks; + +/// +/// Initializes Spark protobuf config. +/// +public interface ISparkConfigLoader { + Task InitializeAsync(CancellationToken cancellationToken); +} diff --git a/src/client/spark_core/Services/SparkConfig.cs b/src/client/spark_core/Services/SparkConfig.cs deleted file mode 100644 index 8f0976363..000000000 --- a/src/client/spark_core/Services/SparkConfig.cs +++ /dev/null @@ -1,6 +0,0 @@ -#nullable enable -namespace Spark.Core.Services; - -public class SparkConfig { - public string? ConfigServer { get; set; } -} diff --git a/src/client/spark_core/Services/SparkConfigLoader.cs b/src/client/spark_core/Services/SparkConfigLoader.cs new file mode 100644 index 000000000..1a0fb9aa4 --- /dev/null +++ b/src/client/spark_core/Services/SparkConfigLoader.cs @@ -0,0 +1,27 @@ +#nullable enable +namespace Spark.Core.Services; + +using bts; + +using mbts.config; + +using Microsoft.Extensions.Options; + +using System.Threading; +using System.Threading.Tasks; + +/// +/// Initializes Spark protobuf config. +/// +class SparkConfigLoader : ISparkConfigLoader { + readonly SparkConfigLoaderOpts config; + + public SparkConfigLoader(IOptions config) + : this(config.Value) { } + + public SparkConfigLoader(SparkConfigLoaderOpts config) => + this.config = config; + + public async Task InitializeAsync(CancellationToken cancellationToken) => + Log.assert(await Configuration.InitializeAsync(config.ConfigServer, cancellationToken)); +} diff --git a/src/client/spark_core/Services/SparkConfigLoaderOpts.cs b/src/client/spark_core/Services/SparkConfigLoaderOpts.cs new file mode 100644 index 000000000..63db5e489 --- /dev/null +++ b/src/client/spark_core/Services/SparkConfigLoaderOpts.cs @@ -0,0 +1,8 @@ +#nullable enable +namespace Spark.Core.Services; + +public class SparkConfigLoaderOpts { + public string? ConfigServer { get; set; } + + public static SparkConfigLoaderOpts Default() => new(); +} diff --git a/src/client/spark_core/Services/SparkConfigLoaderService.cs b/src/client/spark_core/Services/SparkConfigLoaderService.cs new file mode 100644 index 000000000..6dc710242 --- /dev/null +++ b/src/client/spark_core/Services/SparkConfigLoaderService.cs @@ -0,0 +1,25 @@ +#nullable enable +namespace Spark.Core.Services; + +using Microsoft.Extensions.Hosting; + +using System.Threading; +using System.Threading.Tasks; + +/// +/// An facade that initializes Spark protobuf config, logging, and +/// quickbus for both c# and cpp. +/// +public class SparkConfigLoaderService : IHostedService { + readonly ISparkConfigLoader loader; + + public SparkConfigLoaderService(ISparkConfigLoader loader) { + this.loader = loader; + } + + public Task StartAsync(CancellationToken cancellationToken) => + loader.InitializeAsync(cancellationToken); + + public Task StopAsync(CancellationToken cancellationToken) => + Task.CompletedTask; +} diff --git a/src/client/spark_core/Services/SparkCoreServices.cs b/src/client/spark_core/Services/SparkCoreServices.cs index 93e5fe2cb..9cbac82a7 100644 --- a/src/client/spark_core/Services/SparkCoreServices.cs +++ b/src/client/spark_core/Services/SparkCoreServices.cs @@ -8,11 +8,11 @@ using System.Linq; public static class SparkCoreServices { public static IServiceCollection AddSpark(this IServiceCollection services, IConfiguration configurationSection) { - if (services.Any(s => s.ServiceType == typeof(SparkService))) { + if (services.Any(s => s.ServiceType == typeof(SparkConfigLoader))) { throw new InvalidOperationException("SparkService has already been added"); } - return services.Configure(configurationSection) - .AddSingleton() - .AddHostedService(sp => sp.GetRequiredService()); + return services.Configure(configurationSection) + .AddSingleton() + .AddHostedService(sp => sp.GetRequiredService()); } } diff --git a/src/client/spark_core/Services/SparkService.cs b/src/client/spark_core/Services/SparkService.cs deleted file mode 100644 index 8a5e6d9b6..000000000 --- a/src/client/spark_core/Services/SparkService.cs +++ /dev/null @@ -1,32 +0,0 @@ -#nullable enable -namespace Spark.Core.Services; - -using mbts.config; - -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Options; - -using System.Threading; -using System.Threading.Tasks; - -/// -/// An facade that initializes Spark protobuf config, logging, and -/// quickbus for both c# and cpp. -/// -public class SparkService : IHostedService { - readonly SparkConfig config; - - public SparkService(IOptions config) - : this(config.Value) { } - - public SparkService(SparkConfig config) => - this.config = config; - - public Task StartAsync(CancellationToken cancellationToken) { - Configuration.initialize(config.ConfigServer); - return Task.CompletedTask; - } - - public Task StopAsync(CancellationToken cancellationToken) => - Task.CompletedTask; -} diff --git a/src/client/spark_core/config/Configuration.cs b/src/client/spark_core/config/Configuration.cs index 446184364..3def9b1b5 100644 --- a/src/client/spark_core/config/Configuration.cs +++ b/src/client/spark_core/config/Configuration.cs @@ -9,9 +9,12 @@ namespace mbts.config { using System.IO; using System.Runtime.InteropServices; using System.Text; + using System.Threading; + using System.Threading.Tasks; public static class Configuration { static object _config_lock = new(); + static TaskCompletionSource? initTcs; static State _state = State.Uninitialized; @@ -50,19 +53,61 @@ namespace mbts.config { } } - public static void initialize(string? config_source = null) { - if (_state == State.OK) { - return; + public static ValueTask InitializeAsync(string? configSource = null) => + InitializeAsync(configSource, CancellationToken.None); + + public static ValueTask InitializeAsync(CancellationToken cancellationToken) => + InitializeAsync(null, cancellationToken); + + public static ValueTask InitializeAsync(string? config_source, CancellationToken cancellationToken) { + return _state switch { + State.OK => ValueTask.FromResult(true), + State.Error => ValueTask.FromResult(false), + State.Uninitialized => Initialize(), + _ => throw new InvalidOperationException(), + }; + + ValueTask Initialize() { + lock (_config_lock) { + switch (_state) { + case State.OK: + return ValueTask.FromResult(true); + case State.Error: + return ValueTask.FromResult(false); + case State.Uninitialized: + if (initTcs is not null) { + return new(initTcs.Task); + } + initTcs = new(cancellationToken); + break; + default: + throw new InvalidOperationException(); + } + } + // Set 'initTcs' to null at the end so subsequent calls to InitializeAsync re-run BTS.init, + // rather than simply return the same completed task. + // Note: if BTS.init is successful, it won't rerun (see the switch case above). + var task = initTcs.Task; + try { + BTS.init(config_source); + // Guarentee BTS.init runs BEFORE we signal initialization is complete (this code won't + // run concurrently, so a full lock isn't necessary). + Thread.MemoryBarrier(); + _state = State.OK; + initTcs.SetResult(true); + } catch (Exception ex) { + if (ex is ApplicationException) { + lock (_config_lock) { + _state = State.Error; + initTcs.SetException(ex); + initTcs = null; + } + } + return new(false); + } + initTcs = null; + return new(task); } - // Considered doing this async on a background thread but it wouldn't do any good. The next - // guy down the line to ask for config will just block until this completes. - try { - BTS.init(config_source); - } catch (ApplicationException) { - _state = State.Error; - throw; - } - _state = State.OK; } public static void initialize_for_test(string config = "") { @@ -72,7 +117,7 @@ namespace mbts.config { _state = State.OK; } - public static unsafe bool try_initialize(out string error) { + public static bool try_initialize(out string error) { _reset(); try { BTS.init(); @@ -103,12 +148,22 @@ namespace mbts.config { $"Failed to deserialize config '{key}' as {type}: {error}"; static byte[]? _get_buffer(string key_name) { + switch (_state) { + case State.Uninitialized: + if (!InitializeAsync().GetAwaiter().GetResult()) { + return null; + } + break; + case State.Error: + return null; + case State.OK: + break; + default: + throw new InvalidOperationException($"Unexpected state: {_state}"); + } /// The native get_buffer () is not thread /// safe, so this call must be protected from re-entrancy. lock (_config_lock) { - if (_state == State.Uninitialized) { initialize(); } - if (_state == State.Error) { return null; } - int length = 0; byte[] buffer; IntPtr pbuffer = _raw_get_buffer(key_name, ref length);