fix: build was broken, spark.dll being used by another process

Native assets (libspark.dll/so) are copied to the outdir for local
builds, and packed in nupkg files for sdk releases. The dest path in the
package is different from the path in the outdir, so we configured
Copy/Pack as separate msbuild items. However, both had the same
Include="@(NativeWindows/LinuxAsset)", so Pack="false" in the Copy item
was overriding Pack="true" in the Pack item.

.vscode/launch.json

@@ -213,7 +213,7 @@
       "type": "coreclr",
       "request": "launch",
       "program": "",
-      "preLaunchTask": "Build Api Server (Debug)",
+      "preLaunchTask": "Build Spark Sdk (Debug)",
       "windows": {
         "program": "Bts.Spark.Sdk.Demo.exe",
       },

compose.yaml

@@ -108,6 +108,37 @@ services:
       resources:
         limits:
           memory: "${SPARKD__MEMORY:-16gb}"
+  
+  
+  sparkd-multicast-cert:
+    environment:
+      - TZ=America/Chicago
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
+    build:
+      dockerfile: src/app_sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
+    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    network_mode: "host"
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
+    deploy:
+      restart_policy:
+        condition: any
+        delay: 3s
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-16gb}"
+  
   sparkd-tail:
     image: bluetradingsystems/linux_dotnet8
     volumes:

src/Directory.Build.targets

@@ -187,32 +187,24 @@
       Name="Test"
       DependsOnTargets="Build"></Target>
 
-  <ItemGroup Condition="Exists('@(NativeWindowsAsset)')">
-    <Content Include="@(NativeWindowsAsset)">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-      <TargetPath>%(FileName)%(Extension)</TargetPath>
-      <Pack>false</Pack>
-    </Content>
-    <Content Include="@(NativeWindowsAsset)">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-      <TargetPath>runtimes/win-x64/native/%(FileName)%(Extension)</TargetPath>
-      <Pack>true</Pack>
-      <PackagePath>runtimes/win-x64/native</PackagePath>
-    </Content>
-  </ItemGroup>
+  <Target Name="CopyNativeAssetsToOutput" AfterTargets="Build">
+    <Copy SourceFiles="@(NativeWindowsAsset)" DestinationFolder="$(OutDir)" SkipUnchangedFiles="True" Condition="Exists('%(NativeWindowsAsset.FullPath)')" />
+    <Copy SourceFiles="@(NativeLinuxAsset)" DestinationFolder="$(OutDir)" SkipUnchangedFiles="True" Condition="Exists('%(NativeLinuxAsset.FullPath)')" />
+  </Target>
 
-  <ItemGroup Condition="Exists('@(NativeLinuxAsset)')">
-    <Content Include="@(NativeLinuxAsset)">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-      <TargetPath>%(FileName)%(Extension)</TargetPath>
-      <Pack>false</Pack>
-    </Content>
-    <Content Include="@(NativeLinuxAsset)">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-      <TargetPath>runtimes/linux-x64/native/%(FileName)%(Extension)</TargetPath>
-      <Pack>true</Pack>
-      <PackagePath>runtimes/linux-x64/native</PackagePath>
-    </Content>
+  <ItemGroup>
+    <Content Include="@(NativeWindowsAsset)"
+             Condition="Exists('@(NativeWindowsAsset)')"
+             Link="%(NativeWindowsAsset.FileName)%(NativeWindowsAsset.Extension)"
+             CopyToOutputDirectory="PreserveNewest"
+             Pack="true"
+             PackagePath="runtimes/win-x64/native/" />
+    <Content Include="@(NativeLinuxAsset)"
+             Condition="Exists('@(NativeLinuxAsset)')"
+             Link="%(NativeLinuxAsset.FileName)%(NativeLinuxAsset.Extension)"
+             CopyToOutputDirectory="PreserveNewest"
+             Pack="true"
+             PackagePath="runtimes/linux-x64/native/" />
   </ItemGroup>
 
 </Project>

src/app_sparkd/configurations/ophir_cert_live_wb/.env

@@ -0,0 +1,10 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_cert_live_wb
+SPARKD__DATA_DIR=/home/wedbushprod/9000/sparkdata/data
+SPARKD__IMAGE=sparkd_symbols
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_cert_live_wb/requirements.txt
+SPARKD__RUNTIME_CONF_DIR=/home/wedbushprod/9000/sparkdata/conf
+SPARKD__SHM_SIZE=8g
+SPARKD__MEMORY=16g
+SPARKD__CONFIG_PORT=9000
+SPARKD__QUICKBUS_PORT=9001
+SPARKD__SIM_FIX_PORT=9002

src/app_sparkd/configurations/ophir_cert_live_wb/spark.conf

@@ -0,0 +1,140 @@
+DeploymentType {
+  value {
+    type: "dev"
+  }
+}
+PublisherOptions {
+   value {
+      network_interface_a: 'ens65f1'
+      network_interface_b: 'ens65f0'
+      #these get overriden if not entered here
+      heartbeat_ms: 5000
+      heartbeat_boot_seconds: 60
+      skip_options: false
+      hot_initial: 1024
+      hot_growth: 1024
+      cold_initial: 8192
+      cold_growth: 2048
+      channel_prefix: ""
+      disable_heartbeat: false
+   }
+}
+
+CMEOptions {
+  value {
+    feed_engine {
+      name: "mdp3feed"
+      spin_time: 0
+      channels_per_thread: 3
+      data_wait: 10
+      udp_socket_buffer_size: 268435456
+    }
+    channel {
+      id: "*"
+      feed_engine: "mdp3feed"
+      use_tcp_replay: false
+      recovery_queue_size: 640000
+      lost_packet_wait_timeout_ms: 10
+      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
+      #max_gap_recovery_cache_size: 4
+      use_natural_refresh: false
+      join_snapshot_on_gap: false
+    }
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    console {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Info"
+      }
+      subsystem {
+        name: "OrderMan"
+        level: "Info"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    glbx {
+       unique_check_value: 15
+       use_databento: false
+    }
+   # recording {
+   #    venue: VENUE_GLBX
+   #    num_of_seconds_to_run_file_writer: 300
+   #    num_of_seconds_to_run_db_writer : 0
+   #    n_book_levels: 10 
+   #    batch_size: 100 #for the db writer
+   #    enable_book_recording: true
+   #    enable_trade_recording:  true
+   #    enable_bbo_recording: false
+   #    enable_tbbo_recording:  false
+   #    enable_misc_recording: true
+   #}
+ }
+}
+
+JetFIX {
+  value {
+    connection {
+      name: "maw_004_64"
+      venue: VENUE_GLBX
+      socket_connect_host_ip: "69.50.112.177"
+      socket_connect_port: 44040
+      sender_compid: "MAW004"
+      target_compid: "CME"
+      target_subid: "64"
+      cme_market_segment_id: 64
+      start_time: "00:10:00"
+      end_time: "23:55:00"
+      start_day: "sunday"
+      end_day: "friday"
+      seconds_between_hbs: 15
+      sender_subid: "BTS"
+      seconds_to_pause: 2
+      max_connection_attempts: 100
+      use_ilink3: true
+    }
+  }
+}
+OrdermanCfg {
+  value {
+    executor {
+      thread_name: "oman_orders"
+      poll: true 
+      hosted_actor: "MDSource"
+      hosted_actor: "OrderManager"
+      hosted_actor: "ExecutionGateway"
+    }
+    executor {
+      thread_name: "oman_slow"
+            poll: false 
+      hosted_actor: "*"
+    }
+    fallback_cust_order_handling_inst: "Y"
+  }
+}
+NotSimExecution {
+  value {
+    entries {
+      venue: VENUE_GLBX
+    }
+  }
+}
+Ports {
+   value {
+      config_port: 9000
+      qbus_port: 9001
+      glbx_sim_fix_port: 9002
+      ice_sim_fix_port: 9003
+      replicator_port: 9080
+   }
+}

src/app_sparkd/configurations/ophir_demo/.env

@@ -1,5 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_demo
-SPARKD__DATA_DIR=/home/ubuntu/8000/sparkdata/data
-SPARKD__IMAGE=sparkd_symbols
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_demo/requirements.txt
-SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf

src/app_sparkd/configurations/ophir_demo/spark.conf

@@ -1,62 +0,0 @@
-JetFIX {
-  value {
-    connection {
-      name: "glbx_1"
-      venue: VENUE_GLBX
-      max_order_msgs_per_window: 300
-      window_duration_msecs: 1000
-    }
-  }
-}
-
-PublisherOptions {
-  value {
-    network_interface: "0.0.0.0"
-    heartbeat_ms: 5000
-    heartbeat_boot_seconds: 60
-    skip_options: true
-    hot_initial: 1024
-    hot_growth: 1024
-    cold_initial: 8192
-    cold_growth: 2048
-    channel_prefix: ""
-    disable_heartbeat: true
-  }
-}
-
-Logging {
-  value {
-    initial_queue_size: 120001
-    max_queue_growth: 4
-    console {
-      default_level: "Info"
-      subsystem {
-        name: "md"
-        level: "Warn"
-      }
-      subsystem {
-        name: "OrderMan"
-        level: "Info"
-      }
-    }
-    file {
-      default_level: "Info"
-      subsystem {
-        name: "OrderMan"
-        level: "Diag"
-      }
-      subsystem {
-        name: "Http"
-        level: "Diag"
-      }
-    }
-  }
-}
-
-MarketDataConfig {
-  value {
-    replay {
-      venue: VENUE_GLBX
-    }
-  }
-}

src/app_sparkd/configurations/ophir_demo/symbols.conf

@@ -1,8 +0,0 @@
-Symbols {
-  value {
-    venues {
-      name: "VENUE_GLBX"
-      future: "ES"
-    }
-  }
-}

src/app_sparkd/configurations/ophir_prod_live_wb/.env

@@ -1,5 +1,5 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_sim_live_wb
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_prod_live_wb
 SPARKD__DATA_DIR=/home/wedbushprod/8000/sparkdata/data
 SPARKD__IMAGE=sparkd_symbols
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_sim_live_wb/requirements.txt
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_prod_live_wb/requirements.txt
 SPARKD__RUNTIME_CONF_DIR=/home/wedbushprod/8000/sparkdata/conf

src/app_sparkd/configurations/ophir_prod_live_wb/spark.conf

@@ -0,0 +1,152 @@
+DeploymentType {
+  value {
+    type: "prod"
+  }
+}
+PublisherOptions {
+   value {
+      network_interface_a: 'enp193s0f0np0'
+      network_interface_b: 'enp193s0f1np1'
+      #these get overriden if not entered here
+      heartbeat_ms: 5000
+      heartbeat_boot_seconds: 60
+      skip_options: false
+      hot_initial: 1024
+      hot_growth: 1024
+      cold_initial: 8192
+      cold_growth: 2048
+      channel_prefix: ""
+      disable_heartbeat: false
+   }
+}
+
+CMEOptions {
+  value {
+    feed_engine {
+      name: "mdp3feed"
+      spin_time: 0
+      channels_per_thread: 3
+      data_wait: 10
+      udp_socket_buffer_size: 268435456
+    }
+    channel {
+      id: "*"
+      feed_engine: "mdp3feed"
+      use_tcp_replay: false
+      recovery_queue_size: 640000
+      lost_packet_wait_timeout_ms: 10
+      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
+      #max_gap_recovery_cache_size: 4
+      use_natural_refresh: false
+      join_snapshot_on_gap: false
+    }
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    console {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Info"
+      }
+      subsystem {
+        name: "OrderMan"
+        level: "Info"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    glbx {
+       unique_check_value: 15
+       use_databento: false
+    }
+   # recording {
+   #    venue: VENUE_GLBX
+   #    num_of_seconds_to_run_file_writer: 300
+   #    num_of_seconds_to_run_db_writer : 0
+   #    n_book_levels: 10 
+   #    batch_size: 100 #for the db writer
+   #    enable_book_recording: true
+   #    enable_trade_recording:  true
+   #    enable_bbo_recording: false
+   #    enable_tbbo_recording:  false
+   #    enable_misc_recording: true
+   #}
+ }
+}
+
+JetFIX {
+  value {
+    connection {
+      name: "u6p_7qb_54"
+      venue: VENUE_GLBX
+      socket_connect_host_ip: "205.209.201.97"
+      socket_connect_port: 10979
+      sender_compid: "U6P7QB"
+      target_compid: "CME"
+      target_subid: "54"
+      cme_market_segment_id: 54
+      start_time: "00:10:00"
+      end_time: "23:55:00"
+      start_day: "sunday"
+      end_day: "friday"
+      seconds_between_hbs: 15
+      sender_subid: "BTS"
+      seconds_to_pause: 1
+      max_order_msgs_per_window: 300
+      max_connection_attempts: 2
+      use_ilink3: true
+    }
+   connection {
+      name: "u6p_7qb_64"
+      venue: VENUE_GLBX
+      socket_connect_host_ip: "205.209.201.1"
+      socket_connect_port: 10979
+      sender_compid: "U6P7QB"
+      target_compid: "CME"
+      target_subid: "64"
+      cme_market_segment_id: 64
+      start_time: "00:10:00"
+      end_time: "23:55:00"
+      start_day: "sunday"
+      end_day: "friday"
+      seconds_between_hbs: 15
+      sender_subid: "BTS"
+      seconds_to_pause: 1
+      max_connection_attempts: 2
+      max_order_msgs_per_window: 300
+      use_ilink3: true
+    }
+  }
+}
+OrdermanCfg {
+  value {
+    executor {
+      thread_name: "oman_orders"
+      poll: true 
+      hosted_actor: "MDSource"
+      hosted_actor: "OrderManager"
+      hosted_actor: "ExecutionGateway"
+    }
+    executor {
+      thread_name: "oman_slow"
+            poll: false 
+      hosted_actor: "*"
+    }
+    fallback_cust_order_handling_inst: "Y"
+  }
+}
+NotSimExecution {
+  value {
+    entries {
+      venue: VENUE_GLBX
+    }
+  }
+}

src/app_sparkd/configurations/ophir_sim/.env

@@ -1,5 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_sim
-SPARKD__DATA_DIR=/home/tns/8000/sparkdata/data
-SPARKD__IMAGE=sparkd_symbols
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_sim/requirements.txt
-SPARKD__RUNTIME_CONF_DIR=/home/tns/8000/sparkdata/conf

src/app_sparkd/configurations/ophir_sim/spark.conf

@@ -1,104 +0,0 @@
-JetFIX {
-  value {
-    connection {
-      name: "glbx_1"
-      venue: VENUE_GLBX
-      max_order_msgs_per_window: 300
-      window_duration_msecs: 1000
-    }
-  }
-}
-
-PublisherOptions {
-   value {
-      network_interface_a: 'ens3f0np0'
-      network_interface_b: 'ens3f1np1'
-      #these get overriden if not entered here
-      heartbeat_ms: 5000
-      heartbeat_boot_seconds: 60
-      skip_options: false
-      hot_initial: 1024
-      hot_growth: 1024
-      cold_initial: 8192
-      cold_growth: 2048
-      channel_prefix: ""
-      disable_heartbeat: false
-   }
-}
-
-CMEOptions {
-  value {
-    feed_engine {
-      name: "mdp3feed"
-      spin_time: 0
-      channels_per_thread: 3
-      data_wait: 10
-      udp_socket_buffer_size: 268435456
-    }
-    channel {
-      id: "*"
-      feed_engine: "mdp3feed"
-      use_tcp_replay: false
-      recovery_queue_size: 640000
-      lost_packet_wait_timeout_ms: 10
-      onixs_log_mode: CME_ONIXS_LOG_MODE_REGULAR
-      #max_gap_recovery_cache_size: 4
-      use_natural_refresh: false
-      join_snapshot_on_gap: false
-    }
-  }
-}
-
-Logging {
-  value {
-    initial_queue_size: 120001
-    max_queue_growth: 4
-    console {
-      default_level: "Info"
-      subsystem {
-        name: "md"
-        level: "Info"
-      }
-      subsystem {
-        name: "OrderMan"
-        level: "Info"
-      }
-    }
-  }
-}
-
-MarketDataConfig {
-  value {
-    glbx {
-       unique_check_value: 15
-       use_databento: false
-    }
-  }
-}
-
-# JetFIX {
-#    value {
-#     #connection: @JetFIXSimConnection.sim
-#     connection {
-#     name: "mlx_004_54"
-#     venue: VENUE_GLBX
-#     socket_connect_host_ip: "69.50.112.191"
-#     socket_connect_port: 42876
-#     sender_compid: "MLX004"
-#     target_compid: "CME"
-#     target_subid: "54"
-#     cme_market_segment_id: 54
-#     start_time: "00:10:00"
-#     end_time: "23:55:00"
-#     start_day: "sunday"
-#     end_day: "friday"
-#     seconds_between_hbs: 30
-#     sender_subid: "BTS"
-#     use_ilink3: true
-#   #   raw_data: "S4PMA"
-#   #   auto_fix_seq_num: true
-#   }
-#    }
-# }
-
-

src/app_sparkd/configurations/ophir_sim_live_wb/requirements.txt

@@ -1,5 +0,0 @@
-numpy==2.2.6
-pybit==5.11.0
-requests==2.31.0
-websocket-client==1.8.0
-websockets==15.0.1

src/app_sparkd/configurations/ophir_sim_live_wb/spark.conf

@@ -1,116 +0,0 @@
-JetFIX {
-  value {
-    connection {
-      name: "glbx_1"
-      venue: VENUE_GLBX
-      max_order_msgs_per_window: 300
-      window_duration_msecs: 1000
-    }
-  }
-}
-
-PublisherOptions {
-   value {
-      network_interface_a: 'enp193s0f0np0'
-      network_interface_b: 'enp193s0f0np0'
-      #these get overriden if not entered here
-      heartbeat_ms: 5000
-      heartbeat_boot_seconds: 60
-      skip_options: false
-      hot_initial: 1024
-      hot_growth: 1024
-      cold_initial: 8192
-      cold_growth: 2048
-      channel_prefix: ""
-      disable_heartbeat: false
-   }
-}
-
-CMEOptions {
-  value {
-    feed_engine {
-      name: "mdp3feed"
-      spin_time: 0
-      channels_per_thread: 3
-      data_wait: 10
-      udp_socket_buffer_size: 268435456
-    }
-    channel {
-      id: "*"
-      feed_engine: "mdp3feed"
-      use_tcp_replay: false
-      recovery_queue_size: 640000
-      lost_packet_wait_timeout_ms: 10
-      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED:q
-      #max_gap_recovery_cache_size: 4
-      use_natural_refresh: false
-      join_snapshot_on_gap: false
-    }
-  }
-}
-
-Logging {
-  value {
-    initial_queue_size: 120001
-    max_queue_growth: 4
-    console {
-      default_level: "Info"
-      subsystem {
-        name: "md"
-        level: "Info"
-      }
-      subsystem {
-        name: "OrderMan"
-        level: "Info"
-      }
-    }
-  }
-}
-
-MarketDataConfig {
-  value {
-    glbx {
-       unique_check_value: 15
-       use_databento: false
-    }
-    recording{
-       venue: VENUE_GLBX
-       num_of_seconds_to_run_file_writer: 300
-       num_of_seconds_to_run_db_writer : 0
-       n_book_levels: 10 
-       batch_size: 100 #for the db writer
-       enable_book_recording: true
-       enable_trade_recording:  true
-       enable_bbo_recording: false
-       enable_tbbo_recording:  false
-       enable_misc_recording: true
-   }
-  }
-}
-
-# JetFIX {
-#    value {
-#     #connection: @JetFIXSimConnection.sim
-#     connection {
-#     name: "mlx_004_54"
-#     venue: VENUE_GLBX
-#     socket_connect_host_ip: "69.50.112.191"
-#     socket_connect_port: 42876
-#     sender_compid: "MLX004"
-#     target_compid: "CME"
-#     target_subid: "54"
-#     cme_market_segment_id: 54
-#     start_time: "00:10:00"
-#     end_time: "23:55:00"
-#     start_day: "sunday"
-#     end_day: "friday"
-#     seconds_between_hbs: 30
-#     sender_subid: "BTS"
-#     use_ilink3: true
-#   #   raw_data: "S4PMA"
-#   #   auto_fix_seq_num: true
-#   }
-#    }
-# }
-
-

src/client/spark/Modules/Trading/Fills/FillsRowVM.cs

@@ -78,6 +78,9 @@ namespace mbts.spark {
       public Fill.Types.FillStatus fill_status => _fill.FillStatus;
       public string fill_key => _fill.FillKey;
       public string order_key => _order.Key.ToString();
+      // public bool is_aggressor => fill_tuple.is_aggressor;
+      public string display_aggressor_flag_text =>
+         display_aggressor_flag(_fill.FillAggressorFlag);
       public bool is_selected {
          get => _is_selected;
          set {
@@ -105,6 +108,13 @@ namespace mbts.spark {
       AlgoData? _root_algo => fill_tuple.root_algo;
       AlgoDefinitionMetadata? _root_algo_meta => fill_tuple.root_algo_meta;
 
+      public static string display_aggressor_flag(FillAggressorFlag flag) =>
+         flag switch {
+            FillAggressorFlag.Aggressor => "A",
+            FillAggressorFlag.Passive => "P",
+            _ => string.Empty
+         };
+
       public static FillsRowVM update(FillsRowVM r, FillTuple ft) {
          using var _ = _change_detector.detect_changes(r);
          r._badge_fill = null;

src/client/spark/Modules/Trading/Fills/FillsVM.cs

@@ -295,7 +295,7 @@ namespace mbts.spark {
          add_col(
             Columns.IsAggressor,
             DefaultColumnHeaders.IsAggressor,
-            r => r.is_aggressor ? SpecialCharacters.CheckMark : string.Empty,
+            r => FillsRowVM.display_aggressor_flag(r.fill.FillAggressorFlag),
             dv_filter(new[] { false, true }, r => r.is_aggressor),
             DataGridFilterKnownTemplates.YesNo,
             fs => fs.is_aggressor,

src/cs/Bts.Spark.Api/Orders/OrdersClient.cs

@@ -11,6 +11,7 @@ namespace Spark.Api.Orders {
 
    using Spark.Api.Admin;
    using Spark.Api.MarketData;
+   using Spark.Api.Position;
    using Spark.Api.Reactive;
    using Spark.Core.Disposables;
    using Spark.Core.Reactive;
@@ -41,12 +42,11 @@ namespace Spark.Api.Orders {
          _order_service = order_service;
          var scheduler = ThreadPoolScheduler.Instance;
          if(interval is not null)
-            logger.Information("originally OrdersClient polling interval {0}", interval);
+            Log.Api.info($"originally OrdersClient polling interval {interval}");
          interval = TimeSpan.FromMilliseconds(1);
-         logger.Information("OrdersClient polling interval {0}", interval);
+         Log.Api.info($"OrdersClient polling interval {interval}");
          var bootstraps = new DisposableBuilder();
 
-         positions = position_service.subscribei_position_data().Table().Cache();
          _orders = bootstraps.emplace(
             admin_client
                .permission_filter(
@@ -76,6 +76,11 @@ namespace Spark.Api.Orders {
                .Cache()
                .ToObservable());
 
+         var raw_fills = admin_client.permission_filter(
+            order_service.subscribei_fills().Naggle(interval.Value, scheduler),
+            f => f.Account)
+            .Table();
+
          var fill_order_tuples_join = fills.LeftJoin(
             _orders,
             f => f.val.FillKey,
@@ -97,6 +102,12 @@ namespace Spark.Api.Orders {
                .Cache()
                .ToObservable());
 
+         var raw_manual_fills = admin_client
+            .permission_filter(
+               position_service.subscribei_manual_fills().Naggle(interval.Value, scheduler),
+               f => f.AccountId)
+            .Table();
+
          var manual_fill_tuples_account_join = manual_fills_securities.LeftJoin(
             admin_client.Accounts,
             mf => mf.val.RecordId,
@@ -120,6 +131,11 @@ namespace Spark.Api.Orders {
                .Cache()
                .ToObservable());
 
+         positions = bootstraps.emplace(
+            ComputedPositionsDataSource.create(raw_fills, raw_manual_fills)
+               .Cache()
+               .ToObservable());
+
          _dispose = bootstraps;
       }
       public async Task<Response> EnterOrderAsync(

src/cs/Bts.Spark.Api/Position/ComputedPositionsDataSource.cs

@@ -0,0 +1,194 @@
+#nullable enable
+namespace Spark.Api.Position {
+   using bts;
+   using bts.api;
+   using bts.api.core;
+   using bts.api.orderman;
+   using bts.api.position;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Reactive.Linq;
+
+   public static class ComputedPositionsDataSource {
+      public static IObservable<TableUpdate<PositionUnitImmutable, int>> create(
+         IObservable<TableUpdate<FillImmutable, string>> fills,
+         IObservable<TableUpdate<ManualFillImmutable, int>> manual_fills) {
+         var record_ids = new Dictionary<AccountSecurityKey, int>();
+         int next_record_id = 1;
+
+         return Observable.CombineLatest(
+            fills,
+            manual_fills,
+            (fill_update, manual_fill_update) => build_snapshot(
+               fill_update.Index.Values,
+               manual_fill_update.Index.Values)
+               .ToDictionary(PositionUnitImmutable.key_fn))
+            .Table(PositionUnitImmutable.key_fn);
+
+         IEnumerable<PositionUnitImmutable> build_snapshot(
+            IEnumerable<FillImmutable> electronic_fills,
+            IEnumerable<ManualFillImmutable> manual_fill_rows) {
+            var positions = new SortedDictionary<AccountSecurityKey, PositionAccumulator>();
+
+            foreach (var fill in electronic_fills) {
+               if (fill.FillStatus == Fill.Types.FillStatus.Busted ||
+                  fill.SecurityKey == 0 ||
+                  fill.InternalAccountId == 0) {
+                  continue;
+               }
+               var key = new AccountSecurityKey(fill.InternalAccountId, fill.SecurityKey);
+               var acc = get_or_create(positions, key, fill.Account);
+               acc.add_electronic_fill(fill);
+               positions[key] = acc;
+            }
+
+            foreach (var fill in manual_fill_rows) {
+               if (fill.SecurityKey == 0 || fill.AccountId == 0) {
+                  continue;
+               }
+               var key = new AccountSecurityKey(fill.AccountId, fill.SecurityKey);
+               var acc = get_or_create(positions, key, fill.AccountName);
+               acc.add_manual_fill(fill);
+               positions[key] = acc;
+            }
+
+            foreach (var (key, acc) in positions) {
+               if (!record_ids.TryGetValue(key, out var record_id)) {
+                  record_id = next_record_id++;
+                  record_ids[key] = record_id;
+               }
+               yield return acc.to_position(record_id);
+            }
+         }
+
+         static PositionAccumulator get_or_create(
+            IDictionary<AccountSecurityKey, PositionAccumulator> positions,
+            AccountSecurityKey key,
+            string account_name) {
+            if (!positions.TryGetValue(key, out var acc)) {
+               return new PositionAccumulator(key.account_id, key.security_id, account_name);
+            } else if (string.IsNullOrEmpty(acc.account) && !string.IsNullOrEmpty(account_name)) {
+               acc.account = account_name;
+            }
+            return acc;
+         }
+      }
+
+      struct PositionAccumulator {
+         public PositionAccumulator(uint account_id, uint security_id, string account) {
+            this.account_id = account_id;
+            this.security_id = security_id;
+            this.account = account;
+            day = default;
+            sod = default;
+            local = default;
+            admin_day = default;
+            admin_sod = default;
+            raw_day = default;
+         }
+
+         public readonly uint account_id;
+         public readonly uint security_id;
+         public string account;
+         PositionEntryAccumulator day;
+         PositionEntryAccumulator sod;
+         PositionEntryAccumulator local;
+         PositionEntryAccumulator admin_day;
+         PositionEntryAccumulator admin_sod;
+         PositionEntryAccumulator raw_day;
+
+         public void add_electronic_fill(FillImmutable fill) {
+            var qty = fill.Side == Side.Sell ? -fill.LastShares : fill.LastShares;
+            if (is_sod(fill.FillKind)) {
+               sod.add(qty, fill.LastPrice);
+               return;
+            }
+            day.add(qty, fill.LastPrice);
+            raw_day.add(qty, fill.LastPrice);
+         }
+
+         public void add_manual_fill(ManualFillImmutable fill) {
+            var qty = fill.Side == Side.Sell ? -fill.Quantity : fill.Quantity;
+            switch (fill.FillType) {
+               case ManualFill.Types.FillType.AdminSod:
+                  sod.add(qty, fill.Price);
+                  admin_sod.add(qty, fill.Price);
+                  break;
+               case ManualFill.Types.FillType.AdminDay:
+                  day.add(qty, fill.Price);
+                  raw_day.add(qty, fill.Price);
+                  admin_day.add(qty, fill.Price);
+                  break;
+               default:
+                  day.add(qty, fill.Price);
+                  raw_day.add(qty, fill.Price);
+                  local.add(qty, fill.Price);
+                  break;
+            }
+         }
+
+         public PositionUnitImmutable to_position(int record_id) {
+            var sod_entry = sod.to_entry();
+            var day_entry = day.to_entry();
+            var raw_day_entry = raw_day.to_entry();
+            var local_entry = local.to_entry();
+            var admin_sod_entry = admin_sod.to_entry();
+            var admin_day_entry = admin_day.to_entry();
+
+            return new PositionUnitImmutable(new PositionUnit {
+               RecordId = record_id,
+               Account = account,
+               AccountId = account_id,
+               SecurityId = security_id,
+               SodLongPos = sod_entry.LongPos,
+               SodLongAvgPrice = sod_entry.LongAvgPrice,
+               SodShortPos = sod_entry.ShortPos,
+               SodShortAvgPrice = sod_entry.ShortAvgPrice,
+               DayLongPos = day_entry.LongPos,
+               DayLongAvgPrice = day_entry.LongAvgPrice,
+               DayShortPos = day_entry.ShortPos,
+               DayShortAvgPrice = day_entry.ShortAvgPrice,
+               Sod = sod_entry,
+               Day = day_entry,
+               RawDay = raw_day_entry,
+               Local = local_entry,
+               AdminSod = admin_sod_entry,
+               AdminDay = admin_day_entry
+            });
+         }
+
+         static bool is_sod(Fill.Types.FillKind kind) =>
+            kind == Fill.Types.FillKind.Sod ||
+            kind == Fill.Types.FillKind.SodSpreadLeg ||
+            kind == Fill.Types.FillKind.SodStripLeg;
+      }
+
+      struct PositionEntryAccumulator {
+         double long_qty;
+         double long_cost;
+         double short_qty;
+         double short_cost;
+
+         public void add(double signed_qty, double price) {
+            if (signed_qty > 0) {
+               long_qty += signed_qty;
+               long_cost += signed_qty * price;
+            } else if (signed_qty < 0) {
+               var abs_qty = Math.Abs(signed_qty);
+               short_qty += abs_qty;
+               short_cost += abs_qty * price;
+            }
+         }
+
+         public PositionEntry to_entry() =>
+            new() {
+               LongPos = long_qty,
+               LongAvgPrice = long_qty == 0 ? 0 : long_cost / long_qty,
+               ShortPos = short_qty,
+               ShortAvgPrice = short_qty == 0 ? 0 : short_cost / short_qty
+            };
+      }
+   }
+}

src/cs/Bts.Spark.Api/Position/PositionModel.cs

@@ -15,6 +15,7 @@ namespace Spark.Trading {
    using Spark.Api.Data;
    using Spark.Api.MarketData;
    using Spark.Api.Orders;
+   using Spark.Api.Position;
    using Spark.Api.Reactive;
    using Spark.Core;
    using Spark.Core.Data;
@@ -60,13 +61,21 @@ namespace Spark.Trading {
                .Cache()
                .ToObservable());
 
+         var raw_fills = admin_model
+            .permission_filter(
+               order_service.subscribei_fills().Naggle(interval, scheduler),
+               f => f.Account)
+            .Table();
+
+         var raw_manual_fills = admin_model
+            .permission_filter(
+               _position_service.subscribei_manual_fills().Naggle(interval, scheduler),
+               f => f.AccountId)
+            .Table();
+
          on_positions = bootstraps.emplace(
-            admin_model
-               .permission_filter(
-                  _position_service.subscribei_position_data().Naggle(interval, scheduler),
-                  p => p.AccountId)
-               .join(sec_resolver, pos => pos.SecurityId)
-               .Table()
+            ComputedPositionsDataSource.create(raw_fills, raw_manual_fills)
+               .join_table(sec_resolver, pos => pos.SecurityId)
                .Cache()
                .ToObservable());
          var users = admin_model.Users

src/cs/Bts.Spark.Sdk.Demo/TopOfBookQuoter.cs

@@ -41,8 +41,8 @@ class TopOfBookQuoter : IHostedService {
          throw new ArgumentException("Failed to parse venue", nameof(config));
       }
       var quantity = config.Quantity;
-      if (!(quantity > 0)) {
-         throw new ArgumentException("Quantity is not positive", nameof(config));
+      if (quantity == 0 || !double.IsRealNumber(quantity)) {
+         throw new ArgumentException("Quantity must be real and non-zero", nameof(config));
       }
       this.config = new(venue, contract, account, config.OffsetTicks, quantity);
    }
@@ -86,11 +86,12 @@ class TopOfBookQuoter : IHostedService {
       if (!bk.Security.HasValue) {
          return;
       }
-      var bidPrice = bk[Side.Buy, 0].Price;
-      var orderPrice = bidPrice;
       var orderQty = config.Quantity;
+      var bidPrice = bk[orderQty < 0 ? Side.Sell : Side.Buy, 0].Price;
+      orderQty = Math.Abs(orderQty);
+      var orderPrice = bidPrice;
       if (config.OffsetTicks != 0) {
-         orderPrice = bk.Security.Value.tick_factors().tickdown(orderPrice, config.OffsetTicks);
+         orderPrice = bk.Security.Value.tick_factors().tick(orderPrice, config.OffsetTicks);
       }
       if (securityToOrderId.TryGetValue(bk.SecurityId, out var orderId)
          && orders.TryGetValue(orderId, out var orderTree)

src/cs/Bts.Spark.Sdk.Demo/VenueParser.cs

@@ -4,7 +4,7 @@ using bts.api;
 
 public static class VenueParser {
    public static bool TryParse(string? venueStr, out Venue venue) {
-      if (venueStr is null || (!Enum.TryParse(venueStr, false, out venue) && !Enum.TryParse("VENUE_" + venueStr, false, out venue))) {
+      if (venueStr is null || (!Enum.TryParse(venueStr, true, out venue) && !Enum.TryParse("VENUE_" + venueStr, true, out venue))) {
          venue = Venue.Unset;
          return false;
       }

src/cs/Bts.Spark.Sdk/SimpleSparkClient.cs

@@ -271,28 +271,32 @@ public class SimpleSparkClient : ISimpleSparkClient, IDisposable {
    public Task<Response> CreateLimitOrder(string account, Venue venue, string contract, Side side, double quantity, double price, string? tag) {
       var def = buildOrder(account, venue, contract, limitOrderPool);
       def.Side = side;
-      def.Tag = tag;
+      if (tag is not null)
+         def.Tag = tag;
       return ordersClient.TryEnterOrderAsync(def, price, quantity);
    }
 
    public Task<Response> CreateLimitOrder(string account, uint securityId, Side side, double quantity, double price, string? tag) {
       var def = buildOrder(account, securityId, limitOrderPool);
       def.Side = side;
-      def.Tag = tag;
+      if (tag is not null)
+         def.Tag = tag;
       return ordersClient.TryEnterOrderAsync(def, price, quantity);
    }
 
    public Task<Response> CreateMarketOrder(string account, Venue venue, string contract, Side side, double quantity, string? tag) {
       var def = buildOrder(account, venue, contract, marketOrderPool);
       def.Side = side;
-      def.Tag = tag;
+      if (tag is not null)
+         def.Tag = tag;
       return ordersClient.TryEnterOrderAsync(def, double.NaN, quantity);
    }
 
    public Task<Response> CreateMarketOrder(string account, uint securityId, Side side, double quantity, string? tag) {
       var def = buildOrder(account, securityId, marketOrderPool);
       def.Side = side;
-      def.Tag = tag;
+      if (tag is not null)
+         def.Tag = tag;
       return ordersClient.TryEnterOrderAsync(def, double.NaN, quantity);
    }
 

src/lib_spark/spark/config/profiles/default/cme_reference_data.conf

@@ -3,9 +3,9 @@
 # source: tools/cpp_helper/generate-cpps.ps1
 #
 
-CppConfig
-{
-	value
+CppConfig
+{
+	value
 	{
 		cpp_entry { key: "0A" value: 100 }
 		cpp_entry { key: "0B" value: 100 }
@@ -537,6 +537,31 @@ CppConfig
 		cpp_entry { key: "EQ1" value: 50 }
 		cpp_entry { key: "ERL" value: 5 }
 		cpp_entry { key: "ES" value: 50 }
+		cpp_entry { key: "E1A" value: 50}
+		cpp_entry { key: "E1B" value: 50}
+		cpp_entry { key: "E1C" value: 50}
+		cpp_entry { key: "E1D" value: 50}
+		cpp_entry { key: "E2A" value: 50}
+		cpp_entry { key: "E2B" value: 50}
+		cpp_entry { key: "E2C" value: 50}
+		cpp_entry { key: "E2D" value: 50}
+		cpp_entry { key: "E3A" value: 50}
+		cpp_entry { key: "E3B" value: 50}
+		cpp_entry { key: "E3C" value: 50}
+		cpp_entry { key: "E3D" value: 50}
+		cpp_entry { key: "E4A" value: 50}
+		cpp_entry { key: "E4B" value: 50}
+		cpp_entry { key: "E4C" value: 50}
+		cpp_entry { key: "E4D" value: 50}
+		cpp_entry { key: "E5A" value: 50}
+		cpp_entry { key: "E5B" value: 50}
+		cpp_entry { key: "E5C" value: 50}
+		cpp_entry { key: "E5D" value: 50}
+		cpp_entry { key: "EW" value: 50}		
+		cpp_entry { key: "EW1" value: 50}
+		cpp_entry { key: "EW2" value: 50}
+		cpp_entry { key: "EW3" value: 50}
+		cpp_entry { key: "EW4" value: 50}
 		cpp_entry { key: "ES1" value: 50 }
 		cpp_entry { key: "ES2" value: 50 }
 		cpp_entry { key: "ESB" value: 1000 }
@@ -1001,6 +1026,31 @@ CppConfig
 		cpp_entry { key: "NOO" value: 8900 }
 		cpp_entry { key: "NPG" value: 2500 }
 		cpp_entry { key: "NQ" value: 20 }
+      	cpp_entry {key: "Q1A" value: 20}
+      	cpp_entry {key: "Q1B" value: 20}
+      	cpp_entry {key: "Q1C" value: 20}
+      	cpp_entry {key: "Q1D" value: 20}
+      	cpp_entry {key: "Q2A" value: 20}
+      	cpp_entry {key: "Q2B" value: 20}
+      	cpp_entry {key: "Q2C" value: 20}
+      	cpp_entry {key: "Q2D" value: 20}
+      	cpp_entry {key: "Q3A" value: 20}
+      	cpp_entry {key: "Q3B" value: 20}
+      	cpp_entry {key: "Q3C" value: 20}
+      	cpp_entry {key: "Q3D" value: 20}
+      	cpp_entry {key: "Q4A" value: 20}
+      	cpp_entry {key: "Q4B" value: 20}
+      	cpp_entry {key: "Q4C" value: 20}
+      	cpp_entry {key: "Q4D" value: 20}
+      	cpp_entry {key: "Q5A" value: 20}
+      	cpp_entry {key: "Q5B" value: 20}
+      	cpp_entry {key: "Q5C" value: 20}
+      	cpp_entry {key: "Q5D" value: 20}
+      	cpp_entry {key: "QNE" value: 20}
+      	cpp_entry {key: "QN1" value: 20}
+      	cpp_entry {key: "QN2" value: 20}
+      	cpp_entry {key: "QN3" value: 20}
+      	cpp_entry {key: "QN4" value: 20}
 		cpp_entry { key: "NQQ" value: 20 }
 		cpp_entry { key: "NQT" value: 20 }
 		cpp_entry { key: "NR" value: 2500 }
@@ -1506,12 +1556,12 @@ CppConfig
 		cpp_entry { key: "ZT" value: 2000 }
 		cpp_entry { key: "ZW" value: 50 }
 		cpp_entry { key: "ZWT" value: 50 }
-	}
-}
+	}
+}
 
-CmeSymbolMics
-{
-	value
+CmeSymbolMics
+{
+	value
 	{
 		cme_mic { key: "0A" value: NYMEX }
 		cme_mic { key: "0B" value: NYMEX }
@@ -3012,5 +3062,5 @@ CmeSymbolMics
 		cme_mic { key: "ZT" value: CBT }
 		cme_mic { key: "ZW" value: CBT }
 		cme_mic { key: "ZWT" value: CBT }
-	}
+	}
 }

src/lib_spark/spark/orderman/ilink3/IL3Parser.cpp

@@ -70,9 +70,35 @@ bool IL3Parser::parse_common(const T &msg, ExecutionReport *data) {
    data->ice_cl_ord_id = 0;
    data->ice_company_id = 0;
    data->cfg = cfg;
+   data->fill_agg_flag = api::orderman::PASSIVE;
    return true;
 }
 
+template <class T>
+void IL3Parser::parse_fill_aggressor(const T &, ExecutionReport *data) {
+   data->fill_agg_flag = api::orderman::PASSIVE;
+}
+
+template <>
+void IL3Parser::parse_fill_aggressor(
+    const iLinkBinary::ExecutionReportTradeOutright525 &msg,
+    ExecutionReport *data) {
+   data->fill_agg_flag =
+       msg.aggressorIndicator() == iLinkBinary::BooleanFlag::True
+           ? api::orderman::AGGRESSOR
+           : api::orderman::PASSIVE;
+}
+
+template <>
+void IL3Parser::parse_fill_aggressor(
+    const iLinkBinary::ExecutionReportTradeSpread526 &msg,
+    ExecutionReport *data) {
+   data->fill_agg_flag =
+       msg.aggressorIndicator() == iLinkBinary::BooleanFlag::True
+           ? api::orderman::AGGRESSOR
+           : api::orderman::PASSIVE;
+}
+
 template <class T>
 void IL3Parser::parse_trade(const T &msg, ExecutionReport *data) {
    data->exchange_fill_id->copy_from(msg.getExecIDAsStringView());
@@ -103,6 +129,7 @@ void IL3Parser::parse_trade(const T &msg, ExecutionReport *data) {
           Price{price.mantissa() * md::pow10(price.exponent())});
    }
    data->filled_quantity = Quantity{msg.cumQty()};
+   parse_fill_aggressor(msg, data);
 }
 
 bool IL3Parser::has_account() const {

src/lib_spark/spark/orderman/ilink3/IL3Parser.h

@@ -47,6 +47,8 @@ class IL3Parser {
  private:
    template <class T> bool parse_common(const T &, ExecutionReport *);
    template <class T> void parse_trade(const T &, ExecutionReport *);
+   template <class T>
+   void parse_fill_aggressor(const T &, ExecutionReport *);
 };
 
 bool parse_report(const std::string &, IL3Parser &, ExecutionReport *);

src/lib_spark/spark/utils/object_pool.h

@@ -23,7 +23,7 @@ class object_pool {
    object_pool() : name(bts::utils::type_name<T>()) {}
 
    object_pool(std::string name, std::size_t max_count, std::size_t log_freq)
-       : name(std::move(name)), max_count(max_count), log_freq(log_freq) {}
+       : max_count(max_count), log_freq(log_freq),name(std::move(name)) {}
 
    void set_max_count(std::size_t value) { max_count = value; }