From fd9892c6b6066d132a8221470968997305759d81 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 22 Dec 2022 10:11:25 -0500 Subject: [PATCH] fix: Prevent raw_pb_callback premature garbage collection Bugs (fixed): * Startup crash w/ error message: "A callback was made on a garbage collected delegate of type ApiNativeMethods.raw_pb_callback" Cause: The callback is only referenced in unmanaged c++. Without references in managed code, the Garbage Collector deems the delegate eligible for collection before the callback has finished execution. Note: The original solution forced a reference in managed code by storing it in a class-member 'keepalive' dictionary, whose life isn't bound to the [synchronous] scope of the 'send' method. The callback would then remove itself from the dictionary in its last line of code. While this reduced the chance of such an error occurring, occassional crashes persisted (particularly in low-bandwidth environments). I suspect the frequency at which garbage-collector threads run is inversely related to the frequency at which data arrives from the server, as the latter triggers execution of higher-priority threads. Steps to repro: 1. Insert a Thread.Sleep statement (1000 ms should be sufficient) after the call that removes the last reference to the delegate (i.e., make the thread sleep for a significant amount of time in the callback AFTER it becomes eligible for garbage collection). 2. Build, run, and observe crash on startup. Solution: Wrap the callback in another class, and store the wrapper in the dictionary rather than the callback itself. The reason why this works isn't totally clear, however its effectiveness has been verified with the repro (described above). --- .../api/Channels/QuickBusClientChannel.cs | 408 +++++++++--------- 1 file changed, 216 insertions(+), 192 deletions(-) diff --git a/src/client/spark_api/api/Channels/QuickBusClientChannel.cs b/src/client/spark_api/api/Channels/QuickBusClientChannel.cs index 1821ad275..d808a68ed 100644 --- a/src/client/spark_api/api/Channels/QuickBusClientChannel.cs +++ b/src/client/spark_api/api/Channels/QuickBusClientChannel.cs @@ -1,192 +1,216 @@ -using Google.Protobuf.WellKnownTypes; -using Google.Protobuf; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using System.Reactive.Linq; -using System.IO; -using System.Reactive.Disposables; -using mbts; -using System.Threading; - -namespace bts.api.core { - - internal class QuickbusClientChannel : ClientChannel { - IntPtr _client; - - public QuickbusClientChannel(ServiceContext context, string type_name, string qualifier) : base(context) { - _client = ApiNativeMethods.api_client_create(context.native_context, type_name, qualifier); - System.Diagnostics.Trace.Assert(_client != IntPtr.Zero); - } - - public override void Dispose() { - ApiNativeMethods.api_client_destroy(_client); - _client = IntPtr.Zero; - } - - - int _req_id = 0; - Dictionary _keepalive_cache = new Dictionary(); - - unsafe public override Task> send(int method_id, Request request) { - var id = Interlocked.Increment(ref _req_id); - var header = new RequestHeader() { - Method = method_id, - RequestId = id, - Token = _context.token.HasValue ? new AuthToken() { Token = _context.token?.Token } : null - }; - var buffer = header.serialize(request); - var tcs = new TaskCompletionSource>(); - //ApiNativeMethods.api_client_log_error($"Call {id}"); - ApiNativeMethods.raw_pb_callback callback = (rbuffer, rlength, _) => { - //ApiNativeMethods.api_client_log_error($"Reply {id}"); - var res = ApiUtils.deserialize(rbuffer, (long)rlength); - if (res.header.Code == ApiError.Success) { - tcs.TrySetResult(new Result(res.body)); - } - else { - tcs.TrySetResult(new Result(res.header.Code)); - } - lock (_keepalive_cache) { _keepalive_cache.Remove(id); } - }; - System.Diagnostics.Trace.Assert(_client != IntPtr.Zero, "Channel may have been disposed"); - lock (_keepalive_cache) { _keepalive_cache.Add(id, callback); } - ApiNativeMethods.api_client_send(_client, buffer, buffer.Length, callback, IntPtr.Zero); - return tcs.Task; - } - - public override IObservable> subscribe(int method_id, string method, string qualifier, MessageParser parser, Func converter) { - var result = Observable.Create>(async observer => { - IntPtr temp_subscription = IntPtr.Zero; - void unsubscribe_temp_sub() { - // Unsubscribe late to keep qb refcount > 0 continuously - if (temp_subscription != IntPtr.Zero) ApiNativeMethods.api_client_unsubscribe(_client, temp_subscription); - } - - // 'tcs' and 'temp_callback' are defined in the same scope as 'temp_subscription' - // (rather than in the 'while loop' scope) to prevent premature garbage collection - TaskCompletionSource tcs = null; - ApiNativeMethods.raw_subscription_callback temp_callback = - (event_type, changes, change_count, unused) => { - tcs?.TrySetResult(true); - }; - - while (true) { - var response = await check_stream_access(method_id, qualifier); - if (response.success) { - if (response.value.Authorized) { - // success. Break loop and subscribe - break; - } - else { - // Not authorized. return empty stream - unsubscribe_temp_sub(); - observer.OnCompleted(); - return System.Reactive.Disposables.Disposable.Empty; - } - } - else { - // Wait for stream to fire once before trying again - unsubscribe_temp_sub(); - tcs = new TaskCompletionSource(); - temp_subscription = ApiNativeMethods.api_client_subscribe( - _client, method, qualifier, temp_callback, IntPtr.Zero); - await tcs.Task; - } - } - - // Authorized. Stream events - unsubscribe_temp_sub(); - // Ensure 'temp_callback' and 'tcs' aren't Garbage Collected before 'temp_subscription' - // is disposed - GC.KeepAlive(temp_callback); - GC.KeepAlive(tcs); - return SubscribeNative(method, qualifier, parser, converter, observer); - }); - Log.QBus.diag($"Subscribing for sub: '{method}/{qualifier}'"); - return result; - } - private Task> check_stream_access(int method_id, string qualifier) { - var request = new StreamAccessRequest() { Qualifier = qualifier ?? "" }; - return send(method_id, request); - } - - private unsafe IDisposable SubscribeNative(string method, string qualifier, MessageParser parser, Func converter, IObserver> observer) where TParse : IMessage { - bool first_update = true; - // Memory optimization. There is only one update per subscription. - RawUpdate the_update = new RawUpdate(); - ApiNativeMethods.raw_subscription_callback cb = (event_type, changes, change_count, unused) => { - the_update.Clear(); - try { - if (event_type == ApiNativeMethods.EventType.TABLE_UPDATED) { - if (changes != null) { - foreach (var change in changes) { - switch (change.type) { - case ApiNativeMethods.ChangeType.REMOVE: - the_update.removed_keys.Add(change.key); - break; - case ApiNativeMethods.ChangeType.REPLACE: - the_update.changed_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len)))); - the_update.changed_keys.Add(change.key); - break; - case ApiNativeMethods.ChangeType.CREATE: - the_update.added_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len)))); - the_update.added_keys.Add(change.key); - break; - } - } - } - observer.OnNext(the_update); - } - else if (event_type == ApiNativeMethods.EventType.TABLE_CREATED) { - if (!first_update) { - observer.OnNext(the_update.MarkAsReset()); - } - first_update = false; - } - } - catch (Exception e) { - ApiNativeMethods.api_client_log_error($"Exception in subscription callback for sub: '{method}/{qualifier}' \n--\n '{e}'"); - // Sleep to let the log message get asyncronously written to disk - Thread.Sleep(100); - Environment.Exit(-1); - } - }; - var subscription = ApiNativeMethods.api_client_subscribe(_client, method, qualifier, cb, IntPtr.Zero); - return FinalizeableAnonymousDisposable.Create((disposing) => { - Log.QBus.diag($"Unsubscribing for sub: '{method}/{qualifier}'"); - ApiNativeMethods.api_client_unsubscribe(_client, subscription); - subscription = IntPtr.Zero; - GC.KeepAlive(cb); - cb = null; - }); - } - - private class FinalizeableAnonymousDisposable : IDisposable { - private Action _action; - public FinalizeableAnonymousDisposable(Action dispose) { - _action = dispose; - } - public static FinalizeableAnonymousDisposable Create(Action dispose) { return new FinalizeableAnonymousDisposable(dispose); } - ~FinalizeableAnonymousDisposable() { - Dispose(false); - } - public void Dispose() { - GC.SuppressFinalize(this); - Dispose(true); - } - - protected virtual void Dispose(bool disposing) { - var temp = _action; - _action = null; - temp?.Invoke(disposing); - } - } - - public override bool is_sync_call_ok() { - return !ApiNativeMethods.api_client_is_on_qb_thread(_client); - } - } -} +using Google.Protobuf.WellKnownTypes; +using Google.Protobuf; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Reactive.Linq; +using System.IO; +using System.Reactive.Disposables; +using mbts; +using System.Threading; + +namespace bts.api.core { + + internal class QuickbusClientChannel : ClientChannel { + IntPtr _client; + + public QuickbusClientChannel(ServiceContext context, string type_name, string qualifier) : base(context) { + _client = ApiNativeMethods.api_client_create(context.native_context, type_name, qualifier); + System.Diagnostics.Trace.Assert(_client != IntPtr.Zero); + } + + public override void Dispose() { + ApiNativeMethods.api_client_destroy(_client); + _client = IntPtr.Zero; + } + + + int _req_id = 0; + Dictionary _keepalive_cache = new(); + + unsafe public override Task> send(int method_id, Request request) { + var id = Interlocked.Increment(ref _req_id); + var header = new RequestHeader() { + Method = method_id, + RequestId = id, + Token = _context.token.HasValue ? new AuthToken() { Token = _context.token?.Token } : null + }; + var buffer = header.serialize(request); + var tcs = new TaskCompletionSource>(); + //ApiNativeMethods.api_client_log_error($"Call {id}"); + ApiNativeMethods.raw_pb_callback callback = (rbuffer, rlength, _) => { + //ApiNativeMethods.api_client_log_error($"Reply {id}"); + var res = ApiUtils.deserialize(rbuffer, (long)rlength); + if (res.header.Code == ApiError.Success) { + tcs.TrySetResult(new Result(res.body)); + } + else { + tcs.TrySetResult(new Result(res.header.Code)); + } + lock (_keepalive_cache) { + _keepalive_cache.Remove(id); + } + }; + System.Diagnostics.Trace.Assert(_client != IntPtr.Zero, "Channel may have been disposed"); + var cb_wrapper = new PBCallbackWrapper(callback); + lock (_keepalive_cache) { + _keepalive_cache.Add(id, cb_wrapper); + } + ApiNativeMethods.api_client_send( + _client, buffer, buffer.Length, cb_wrapper.callback, IntPtr.Zero); + return tcs.Task; + } + + public override IObservable> subscribe(int method_id, string method, string qualifier, MessageParser parser, Func converter) { + var result = Observable.Create>(async observer => { + IntPtr temp_subscription = IntPtr.Zero; + void unsubscribe_temp_sub() { + // Unsubscribe late to keep qb refcount > 0 continuously + if (temp_subscription != IntPtr.Zero) ApiNativeMethods.api_client_unsubscribe(_client, temp_subscription); + } + + // 'tcs' and 'temp_callback' are defined in the same scope as 'temp_subscription' + // (rather than in the 'while loop' scope) to prevent premature garbage collection + TaskCompletionSource tcs = null; + ApiNativeMethods.raw_subscription_callback temp_callback = + (event_type, changes, change_count, unused) => { + tcs?.TrySetResult(true); + }; + + while (true) { + var response = await check_stream_access(method_id, qualifier); + if (response.success) { + if (response.value.Authorized) { + // success. Break loop and subscribe + break; + } + else { + // Not authorized. return empty stream + unsubscribe_temp_sub(); + observer.OnCompleted(); + return System.Reactive.Disposables.Disposable.Empty; + } + } + else { + // Wait for stream to fire once before trying again + unsubscribe_temp_sub(); + tcs = new TaskCompletionSource(); + temp_subscription = ApiNativeMethods.api_client_subscribe( + _client, method, qualifier, temp_callback, IntPtr.Zero); + await tcs.Task; + } + } + + // Authorized. Stream events + unsubscribe_temp_sub(); + // Ensure 'temp_callback' and 'tcs' aren't Garbage Collected before 'temp_subscription' + // is disposed + GC.KeepAlive(temp_callback); + GC.KeepAlive(tcs); + return SubscribeNative(method, qualifier, parser, converter, observer); + }); + Log.QBus.diag($"Subscribing for sub: '{method}/{qualifier}'"); + return result; + } + private Task> check_stream_access(int method_id, string qualifier) { + var request = new StreamAccessRequest() { Qualifier = qualifier ?? "" }; + return send(method_id, request); + } + + private unsafe IDisposable SubscribeNative(string method, string qualifier, MessageParser parser, Func converter, IObserver> observer) where TParse : IMessage { + bool first_update = true; + // Memory optimization. There is only one update per subscription. + RawUpdate the_update = new RawUpdate(); + ApiNativeMethods.raw_subscription_callback cb = (event_type, changes, change_count, unused) => { + the_update.Clear(); + try { + if (event_type == ApiNativeMethods.EventType.TABLE_UPDATED) { + if (changes != null) { + foreach (var change in changes) { + switch (change.type) { + case ApiNativeMethods.ChangeType.REMOVE: + the_update.removed_keys.Add(change.key); + break; + case ApiNativeMethods.ChangeType.REPLACE: + the_update.changed_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len)))); + the_update.changed_keys.Add(change.key); + break; + case ApiNativeMethods.ChangeType.CREATE: + the_update.added_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len)))); + the_update.added_keys.Add(change.key); + break; + } + } + } + observer.OnNext(the_update); + } + else if (event_type == ApiNativeMethods.EventType.TABLE_CREATED) { + if (!first_update) { + observer.OnNext(the_update.MarkAsReset()); + } + first_update = false; + } + } + catch (Exception e) { + ApiNativeMethods.api_client_log_error($"Exception in subscription callback for sub: '{method}/{qualifier}' \n--\n '{e}'"); + // Sleep to let the log message get asyncronously written to disk + Thread.Sleep(100); + Environment.Exit(-1); + } + }; + var subscription = ApiNativeMethods.api_client_subscribe(_client, method, qualifier, cb, IntPtr.Zero); + return FinalizeableAnonymousDisposable.Create((disposing) => { + Log.QBus.diag($"Unsubscribing for sub: '{method}/{qualifier}'"); + ApiNativeMethods.api_client_unsubscribe(_client, subscription); + subscription = IntPtr.Zero; + GC.KeepAlive(cb); + cb = null; + }); + } + + private class FinalizeableAnonymousDisposable : IDisposable { + private Action _action; + public FinalizeableAnonymousDisposable(Action dispose) { + _action = dispose; + } + public static FinalizeableAnonymousDisposable Create(Action dispose) { return new FinalizeableAnonymousDisposable(dispose); } + ~FinalizeableAnonymousDisposable() { + Dispose(false); + } + public void Dispose() { + GC.SuppressFinalize(this); + Dispose(true); + } + + protected virtual void Dispose(bool disposing) { + var temp = _action; + _action = null; + temp?.Invoke(disposing); + } + } + + public override bool is_sync_call_ok() { + return !ApiNativeMethods.api_client_is_on_qb_thread(_client); + } + + /// + /// A simple wrapper that can be used to prevent the premature garbage collection of a + /// delegate. + /// + /// + /// This is part of a workaround for an apparent bug in csharp's GarbageCollector, where the + /// final reference to a delegate instance mustn't be removed until any and all execution has + /// finished (even if the reference is removed by the delegate's last statemwent, a + /// garbage-collector race may yield the infamous error, "A callback was made on a garbage + /// collected delegate of type..."). + /// + class PBCallbackWrapper { + public readonly ApiNativeMethods.raw_pb_callback callback; + + public PBCallbackWrapper(ApiNativeMethods.raw_pb_callback callback) => + this.callback = callback; + } + } +}