Compare commits

..
453 changed files with 9624 additions and 17060 deletions

No files matched your search

-122
View File
@@ -1,122 +0,0 @@
FROM ubuntu:22.04
ARG DEBIAN_FRONTEND=noninteractive
# Note installations that take a while (more than 10s)
RUN apt-get update
RUN apt-get install -y ca-certificates
RUN apt-get install -y ccache
RUN apt-get install -y curl
RUN apt-get install -y fish
RUN apt-get install -y gdb
# git-all: ~250s
RUN apt-get install -y git-all
RUN apt-get install -y gnupg
# liblttng-ust-dev: ~30s
RUN apt-get install -y liblttng-ust-dev
RUN apt-get install -y libssh2-1-dev
RUN apt-get install -y locales
RUN apt-get install -y ninja-build
# openssh-server: ~40s
RUN apt-get install -y openssh-server
RUN apt-get install -y pkg-config
RUN apt-get install -y software-properties-common
RUN apt-get install -y sudo
RUN apt-get install -y tzdata
RUN apt-get install -y vim
RUN apt-get install -y zip
# Python: ~30s
RUN apt-get install -y \
python3-pip \
python3-venv \
&& python3 -m pip install --upgrade pip
# GCC
ARG gcc_major=13
RUN \
add-apt-repository ppa:ubuntu-toolchain-r/test \
&& apt-get update \
&& apt-get install -y g++-$gcc_major
# Clang
ARG clang_major=17
RUN \
curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
| gpg --dearmor - \
| tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
&& echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
| tee -a "/etc/apt/sources.list.d/llvm.list" \
&& apt-get update \
&& apt-get install -y \
clang-$clang_major \
clang-format-$clang_major \
clangd-$clang_major
# CMake
RUN \
curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
| gpg --dearmor - \
| tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
&& echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
| tee /etc/apt/sources.list.d/kitware.list > /dev/null \
&& apt-get update \
&& apt-get install -y cmake
# Dotnet
RUN apt-get update && apt-get install -y dotnet7
# Node
ARG node_major=18
RUN mkdir -p /etc/apt/keyrings \
&& curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
| gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
&& echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
| tee /etc/apt/sources.list.d/nodesource.list \
&& apt-get update \
&& apt-get install -y nodejs
# Emumba (deps)
RUN apt-get update && apt-get -y install rapidjson-dev
# Emumba
COPY emumba /emumba
RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
&& rm -r /emumba
# Pcap
RUN apt-get update && apt-get install -y libpcap0.8-dev
# Set up default executables (update-alternatives)
RUN \
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
# Add compiler symlinks for ccache
RUN \
ln -s /usr/bin/ccache /usr/local/bin/clang; \
ln -s /usr/bin/ccache /usr/local/bin/clang++; \
ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
ln -s /usr/bin/ccache /usr/local/bin/gcc; \
ln -s /usr/bin/ccache /usr/local/bin/g++
RUN locale-gen en_US.UTF-8
ARG docker_user="bts"
ARG docker_uid="1000"
ARG docker_pass="bts"
RUN \
groupadd -g "$docker_uid" "$docker_user" \
&& useradd \
-m \
-u "$docker_uid" \
-g "$docker_uid" \
-G sudo \
--password "$(openssl passwd -1 "$docker_pass")" \
"$docker_user"
-48
View File
@@ -1,48 +0,0 @@
{
"dockerFile": "Dockerfile",
// Creates local dir to mount to HOME in docker
"initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
"hostRequirements": {
"memory": "4gb",
"storage": "6gb"
},
"postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
"containerUser": "bts",
"workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
"workspaceFolder": "/home/bts/spark",
"updateRemoteUserUID": true,
"shutdownAction": "none",
// Note: If running multiple containers on the same host, each must use a
// unique set of ports. Usually we just add 1000 to each port as needed. For
// example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
"runArgs": [
"--shm-size",
"2gb",
"-p",
"8000:8000",
"-p",
"8001:8001",
"-p",
"8002:8002",
"-p",
"8003:8003"
],
"mounts": [
"source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
],
"customizations": {
"vscode": {
"settings": {
"clangd.arguments": [
"--compile-commands-dir=${workspaceFolder}/build/clang/debug"
]
},
"extensions": [
"eamodio.gitlens",
"llvm-vs-code-extensions.vscode-clangd",
"ms-vscode.cmake-tools",
"ms-vscode.cpptools"
]
}
}
}
@@ -1,4 +0,0 @@
#!/usr/bin/env bash
this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"
+3 -3
View File
@@ -47,13 +47,13 @@ trace_data/
/src/.clang-format /src/.clang-format
/src/server/sparkd/cmds/spartan/analysis/.ipynb_checkpoints /src/server/sparkd/cmds/spartan/analysis/.ipynb_checkpoints
/src/server/sparkd/cmds/spartan/analysis/data /src/server/sparkd/cmds/spartan/analysis/data
.tmp/ *.tmp/*
.venv*/ /.venv*/*
.cache/ .cache/
/src/Spark.Admin.WebApp/tsconfig.*.g.json /src/Spark.Admin.WebApp/tsconfig.*.g.json
/src/Spark.Admin.WebApp/src/bin/ /src/Spark.Admin.WebApp/src/bin/
**/CMakeDeps.cmake **/CMakeDeps.cmake
**/CMakeDeps_win.cmake **/CMakeDeps_win.cmake
.devcontainer/ **/devcontainer.json
*.Generated.cs *.Generated.cs
*.lst *.lst
+3 -3
View File
@@ -113,8 +113,8 @@
"description": "Debug build compiled w/ gcc", "description": "Debug build compiled w/ gcc",
"binaryDir": "${sourceDir}/build/gcc/debug", "binaryDir": "${sourceDir}/build/gcc/debug",
"cacheVariables": { "cacheVariables": {
"CMAKE_C_COMPILER": "gcc-12", "CMAKE_C_COMPILER": "gcc",
"CMAKE_CXX_COMPILER": "g++-12", "CMAKE_CXX_COMPILER": "g++",
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG", "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd" "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
} }
@@ -143,7 +143,7 @@
"cacheVariables": { "cacheVariables": {
"CMAKE_C_COMPILER": "gcc", "CMAKE_C_COMPILER": "gcc",
"CMAKE_CXX_COMPILER": "g++", "CMAKE_CXX_COMPILER": "g++",
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3", "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd" "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
} }
}, },
+2 -8
View File
@@ -48,15 +48,9 @@ RUN apt-get update && apt-get install -y \
# cmake # cmake
RUN \ RUN \
curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \ curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
| gpg --dearmor - \ | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg > /dev/null \
| tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \ && apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
&& echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
| tee /etc/apt/sources.list.d/kitware.list > /dev/null \
&& apt-get update \ && apt-get update \
&& rm /usr/share/keyrings/kitware-archive-keyring.gpg \
&& apt-get install -y kitware-archive-keyring \
&& echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs)-rc main" \
| tee -a /etc/apt/sources.list.d/kitware.list >/dev/null \
&& apt-get install -y cmake && apt-get install -y cmake
# node # node
+2 -3
View File
@@ -1,7 +1,6 @@
{ {
"image": "bluetradingsystems/sparkd-dev:latest", "image": "bluetradingsystems/sparkd-dev:latest",
// Creates local dir to mount to HOME in docker "initializeCommand": "${localWorkspaceFolder}/docker/sparkd/dev/devcontainer_initialize.sh",
"initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
"hostRequirements": { "hostRequirements": {
"memory": "4gb", "memory": "4gb",
"storage": "6gb" "storage": "6gb"
@@ -28,6 +27,6 @@
"8003:8003" "8003:8003"
], ],
"mounts": [ "mounts": [
"source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind" "source=${localEnv:HOME}/.sparkd-dev/mounts/sparkd-dev,target=/home/bts,type=bind"
] ]
} }
+3
View File
@@ -0,0 +1,3 @@
#!/usr/bin/env bash
mkdir -p "$HOME/.sparkd-dev/mounts/sparkd-dev"
+6
View File
@@ -0,0 +1,6 @@
cd ~/spark/build/clang/debug/; cmake --build . --target install; rm -rf ~/spark/install/Debug/bin.zip; tar -zcvf ~/spark/install/Debug/bin.zip ~/spark/install/Debug/bin; scp ~/spark/install/Debug/bin.zip bts@10.1.10.31:~/13000/; cd ~/spark/
cd ~/13000/; tar -xf bin.zip; mv home/bts/spark/install/Debug/bin/* sparkbin/; rm -rf ~/13000/home/; docker container stop spark_13000; docker container start spark_13000
cd ~/13000/; tar -xf bin.zip; mv home/bts/spark/install/Debug/bin/* sparkbin/; rm -rf ~/13000/home/; cp -r sparkbin ~/.sparkd-dev/mounts/sparkd_build/;
ES SECURITYTYPE_FUTURE at VENUE_GLBX
./sparkd md dump --symbol ES --type future --venue glbx --platform mdp3onixsex
BT5u53r
@@ -8,7 +8,7 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="Google.Protobuf" Version="3.24.3" /> <PackageReference Include="Google.Protobuf" Version="3.24.3" />
<PackageReference Include="YaccLexTools" Version="1.2.2" /> <PackageReference Include="YaccLexTools" Version="1.2.0" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -1,24 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="YaccLexTools" Version="1.2.2" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
<ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
</ItemGroup>
<ItemGroup Label="ColumnExpression">
<Scanner Include="ColumnExpressionTokens.lex" />
<Parser Include="ColumnExpressionGrammar.y" />
</ItemGroup>
</Project>
Whitespace-only changes.
@@ -1,62 +0,0 @@
namespace Bts.ColumnExpressions;
public class ColumnExpression {
public ColumnExpression(string name, Type? type = null, object? value = null) {
Name = name;
Type = type
?? value?.GetType()
?? throw new ArgumentException(
"Type must be specified explicitly when value is null", nameof(type));
Value = value
?? DBNull.Value;
}
public string Name { get; }
public Type Type { get; }
public object Value { get; }
public static ColumnExpression Create<T>(string name, T value, Type? type = null) {
// Try to convert value to 'type' to verify they are compatible
// Note: Can't use `System.Type.IsAssignableFrom` because it doesn't
// account for implicit conversions (e.g., int -> long), just inheritance.
if (type is not null && !ConvertUtils.try_convert(type, value, out _)) {
throw new ArgumentException(
$"Type '{type}' is not assignable from value type '{value?.GetType()}'");
}
return new(name, type ?? typeof(T), value);
}
internal static Type ParseType(string typeId) {
return Candidates().ExceptNull().FirstOrDefault() ??
throw new ArgumentException($"Failed to parse Type from '{typeId}'", nameof(typeId));
IEnumerable<Type?> Candidates() {
if (typeId.Length == 0) {
yield break;
}
yield return Type.GetType(typeId);
yield return Type.GetType($"System.{typeId}");
if (typeId.Length > 1) {
yield return Type.GetType($"System.{char.ToUpper(typeId[0]) + typeId[1..]}");
}
yield return typeId.ToLower() switch {
"float64" or "double" or "real64" or "number" => typeof(double),
"float32" or "single" or "real32" or "real" or "float" => typeof(float),
"int16" or "short" or "integer16" or "signed16" => typeof(short),
"int32" or "int" or "integer" or "integer32" or "signed" => typeof(int),
"int64" or "long" or "integer64" or "signed64" => typeof(long),
"uint16" or "ushort" or "unsigned16" => typeof(ushort),
"uint32" or "uint" or "unsigned" => typeof(uint),
"uint64" or "ulong" or "unsigned64" => typeof(ulong),
"char" => typeof(char),
"byte" or "uint8" => typeof(byte),
"bool" or "boolean" => typeof(bool),
"string" or "text" => typeof(string),
"date" or "datetime" => typeof(DateTime),
_ => null
};
}
}
}
@@ -1,49 +0,0 @@
%namespace Bts.ColumnExpressions
%partial
%visibility internal
%tokentype ColumnExpressionToken
%parsertype ColumnExpressionParser
%union {
public int Integer;
public double Real;
public bool Boolean;
public string String;
public string Identifier;
public string TypeId;
public DateTime DateTime;
public Tuple<string, Type> ColumnDefinition;
public ColumnExpression ColumnExpression;
public ImmutableList<ColumnExpression> ColumnExpressions;
}
%token BOOLEAN_CONSTANT, INTEGER_CONSTANT, REAL_CONSTANT, DATE_TIME_CONSTANT, STRING_CONSTANT
%token IDENTIFIER, EQUALS, TYPE_ID, OF_TYPE
%nonassoc EQUALS
%type <ColumnDefinition> ColumnDefinition
%type <ColumnExpression> ColumnExpression
%type <ColumnExpressions> ColumnExpressions
%%
ColumnExpressions :
ColumnExpression { $$ = ImmutableList.CreateRange(new[] { $1 }); }
| ColumnExpressions ColumnExpression { $$ = $1.Add($2); }
;
ColumnExpression :
ColumnDefinition EQUALS INTEGER_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Integer, $1.Item2); }
| ColumnDefinition EQUALS REAL_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Real, $1.Item2); }
| ColumnDefinition EQUALS DATE_TIME_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.DateTime, $1.Item2); }
| ColumnDefinition EQUALS BOOLEAN_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Boolean, $1.Item2); }
| ColumnDefinition EQUALS STRING_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.String, $1.Item2); }
| ColumnDefinition { $$ = new($1.Item1, $1.Item2); }
;
ColumnDefinition :
IDENTIFIER { $$ = new($1.Identifier, null); }
| IDENTIFIER OF_TYPE IDENTIFIER { $$ = new($1.Identifier, ColumnExpression.ParseType($3.Identifier)); }
| IDENTIFIER OF_TYPE TYPE_ID { $$ = new($1.Identifier, ColumnExpression.ParseType($3.TypeId)); }
;
@@ -1,18 +0,0 @@
namespace Bts.ColumnExpressions;
using System.IO;
using System.Text;
public class ColumnExpressionInterpreter {
public static IList<ColumnExpression> Evaluate(Stream stream) {
var parser = new ColumnExpressionParser();
var res = parser.Parse(stream);
return res?.ColumnExpressions ?? ImmutableList<ColumnExpression>.Empty.Add(
res?.ColumnExpression ?? throw new Exception("Failed to parse column(s) expression"));
}
public static IList<ColumnExpression> Evaluate(string str) {
using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
return Evaluate(ms);
}
}
@@ -1,18 +0,0 @@
namespace Bts.ColumnExpressions;
using System.Text;
partial class ColumnExpressionParser {
public ColumnExpressionParser() : base(null) { }
public ValueType? Parse(string s) {
using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
return Parse(ms);
}
public ValueType? Parse(Stream stream) {
Scanner = new ColumnExpressionScanner(stream);
Parse();
return CurrentSemanticValue;
}
}
@@ -1,9 +0,0 @@
namespace Bts.ColumnExpressions;
partial class ColumnExpressionScanner {
public override void yyerror(string format, params object[] args) {
base.yyerror(format, args);
Console.WriteLine(format, args);
Console.WriteLine();
}
}
@@ -1,85 +0,0 @@
%namespace Bts.ColumnExpressions
%scannertype ColumnExpressionScanner
%visibility internal
%tokentype ColumnExpressionToken
%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers
/* Literal Constants */
Letter [a-zA-Z]
BooleanConstant true|false
StringConstant '([^\'\\]*|\\.)*'
DateTimeConstant ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
IntegerConstant -?[0-9]+
NullConstant 'null'
RealConstant -?[0-9]+\.[0-9]+
/* Relational Operators */
Equals =
OfType /
Identifier {Letter}({Letter}*[0-9]*_*)*
NamespacePrefix {Identifier}\.
TypeId {NamespacePrefix}*{Identifier}
/* Utility Tokens */
OpenParenthesis \(
CloseParenthesis \)
Comma ,
QuoteOpen \[
QuoteClose \]
/* Comments */
LineComment \/\/[^\n\r]*(\n|\r|\r\n)
%%
{Equals} {
return (int)ColumnExpressionToken.EQUALS;
}
{OfType} {
return (int)ColumnExpressionToken.OF_TYPE;
}
{StringConstant} {
yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'","'");
return (int)ColumnExpressionToken.STRING_CONSTANT;
}
{BooleanConstant} {
yylval.Boolean = yytext.ToLower() switch {
"true" => true,
"false" => false,
_ => throw new Exception(
string.Format("Expected yytext to be 'true' or 'false' (actually '{0}')", yytext))
};
return (int)ColumnExpressionToken.BOOLEAN_CONSTANT;
}
{DateTimeConstant} {
yylval.DateTime = DateTime.Parse(yytext);
return (int)ColumnExpressionToken.DATE_TIME_CONSTANT;
}
{IntegerConstant} {
yylval.Integer = int.Parse(yytext);
return (int)ColumnExpressionToken.INTEGER_CONSTANT;
}
{RealConstant} {
yylval.Real = double.Parse(yytext);
return (int)ColumnExpressionToken.REAL_CONSTANT;
}
{Identifier} {
yylval.Identifier = yytext;
return (int)ColumnExpressionToken.IDENTIFIER;
}
{TypeId} {
yylval.TypeId = yytext;
return (int)ColumnExpressionToken.TYPE_ID;
}
%%
@@ -1,720 +0,0 @@
// ==============================================================
// <auto-generated>
// This code automatically produced from an embedded resource.
// Do not edit this file, or it will become incompatible with
// the specification from which it was generated.
// </auto-generated>
// ==============================================================
using System;
using System.IO;
using System.Text;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Runtime.Serialization;
using System.Globalization;
namespace QUT.GplexBuffers
{
// Code copied from GPLEX embedded resource
[Serializable]
internal class BufferException : Exception
{
public BufferException() { }
public BufferException(string message) : base(message) { }
public BufferException(string message, Exception innerException)
: base(message, innerException) { }
protected BufferException(SerializationInfo info, StreamingContext context)
: base(info, context) { }
}
internal abstract class ScanBuff
{
private string fileNm;
public const int EndOfFile = -1;
public const int UnicodeReplacementChar = 0xFFFD;
public bool IsFile { get { return (fileNm != null); } }
public string FileName { get { return fileNm; } set { fileNm = value; } }
public abstract int Pos { get; set; }
public abstract int Read();
public virtual void Mark() { }
public abstract string GetString(int begin, int limit);
public static ScanBuff GetBuffer(string source)
{
return new StringBuffer(source);
}
public static ScanBuff GetBuffer(IList<string> source)
{
return new LineBuffer(source);
}
#if (!NOFILES)
public static ScanBuff GetBuffer(Stream source)
{
return new BuildBuffer(source);
}
#if (!BYTEMODE)
public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
{
return new BuildBuffer(source, fallbackCodePage);
}
#endif // !BYTEMODE
#endif // !NOFILES
}
#region Buffer classes
// ==============================================================
// ===== Definitions for various ScanBuff derived classes ====
// ==============================================================
// =============== String input ================
// ==============================================================
/// <summary>
/// This class reads characters from a single string as
/// required, for example, by Visual Studio language services
/// </summary>
sealed class StringBuffer : ScanBuff
{
string str; // input buffer
int bPos; // current position in buffer
int sLen;
public StringBuffer(string source)
{
this.str = source;
this.sLen = source.Length;
this.FileName = null;
}
public override int Read()
{
if (bPos < sLen) return str[bPos++];
else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line
else { bPos++; return EndOfFile; } // two strikes and you're out!
}
public override string GetString(int begin, int limit)
{
// "limit" can be greater than sLen with the BABEL
// option set. Read returns a "virtual" EOL if
// an attempt is made to read past the end of the
// string buffer. Without the guard any attempt
// to fetch yytext for a token that includes the
// EOL will throw an index exception.
if (limit > sLen) limit = sLen;
if (limit <= begin) return "";
else return str.Substring(begin, limit - begin);
}
public override int Pos
{
get { return bPos; }
set { bPos = value; }
}
public override string ToString() { return "StringBuffer"; }
}
// ==============================================================
// The LineBuff class contributed by Nigel Horspool,
// nigelh@cs.uvic.cs
// ==============================================================
sealed class LineBuffer : ScanBuff
{
IList<string> line; // list of source lines from a file
int numLines; // number of strings in line list
string curLine; // current line in that list
int cLine; // index of current line in the list
int curLen; // length of current line
int curLineStart; // position of line start in whole file
int curLineEnd; // position of line end in whole file
int maxPos; // max position ever visited in whole file
int cPos; // ordinal number of code in source
// Constructed from a list of strings, one per source line.
// The lines have had trailing '\n' characters removed.
public LineBuffer(IList<string> lineList)
{
line = lineList;
numLines = line.Count;
cPos = curLineStart = 0;
curLine = (numLines > 0 ? line[0] : "");
maxPos = curLineEnd = curLen = curLine.Length;
cLine = 1;
FileName = null;
}
public override int Read()
{
if (cPos < curLineEnd)
return curLine[cPos++ - curLineStart];
if (cPos++ == curLineEnd)
return '\n';
if (cLine >= numLines)
return EndOfFile;
curLine = line[cLine];
curLen = curLine.Length;
curLineStart = curLineEnd + 1;
curLineEnd = curLineStart + curLen;
if (curLineEnd > maxPos)
maxPos = curLineEnd;
cLine++;
return curLen > 0 ? curLine[0] : '\n';
}
// To speed up searches for the line containing a position
private int cachedPosition;
private int cachedIxdex;
private int cachedLineStart;
// Given a position pos within the entire source, the results are
// ix -- the index of the containing line
// lstart -- the position of the first character on that line
private void findIndex(int pos, out int ix, out int lstart)
{
if (pos >= cachedPosition)
{
ix = cachedIxdex; lstart = cachedLineStart;
}
else
{
ix = lstart = 0;
}
while (ix < numLines)
{
int len = line[ix].Length + 1;
if (pos < lstart + len) break;
lstart += len;
ix++;
}
cachedPosition = pos;
cachedIxdex = ix;
cachedLineStart = lstart;
}
public override string GetString(int begin, int limit)
{
if (begin >= maxPos || limit <= begin) return "";
int endIx, begIx, endLineStart, begLineStart;
findIndex(begin, out begIx, out begLineStart);
int begCol = begin - begLineStart;
findIndex(limit, out endIx, out endLineStart);
int endCol = limit - endLineStart;
string s = line[begIx];
if (begIx == endIx)
{
// the usual case, substring all on one line
return (endCol <= s.Length) ?
s.Substring(begCol, endCol - begCol)
: s.Substring(begCol) + "\n";
}
// the string spans multiple lines, yuk!
StringBuilder sb = new StringBuilder();
if (begCol < s.Length)
sb.Append(s.Substring(begCol));
for (; ; )
{
sb.Append("\n");
s = line[++begIx];
if (begIx >= endIx) break;
sb.Append(s);
}
if (endCol <= s.Length)
{
sb.Append(s.Substring(0, endCol));
}
else
{
sb.Append(s);
sb.Append("\n");
}
return sb.ToString();
}
public override int Pos
{
get { return cPos; }
set
{
cPos = value;
findIndex(cPos, out cLine, out curLineStart);
// cLine should be the *next* line after curLine.
curLine = (cLine < numLines ? line[cLine++] : "");
curLineEnd = curLineStart + curLine.Length;
}
}
public override string ToString() { return "LineBuffer"; }
}
#if (!NOFILES)
// ==============================================================
// ===== class BuildBuff : for unicode text files ========
// ==============================================================
class BuildBuffer : ScanBuff
{
// Double buffer for char stream.
class BufferElement
{
StringBuilder bldr = new StringBuilder();
StringBuilder next = new StringBuilder();
int minIx;
int maxIx;
int brkIx;
bool appendToNext;
internal BufferElement() { }
internal int MaxIndex { get { return maxIx; } }
// internal int MinIndex { get { return minIx; } }
internal char this[int index]
{
get
{
if (index < minIx || index >= maxIx)
throw new BufferException("Index was outside data buffer");
else if (index < brkIx)
return bldr[index - minIx];
else
return next[index - brkIx];
}
}
internal void Append(char[] block, int count)
{
maxIx += count;
if (appendToNext)
this.next.Append(block, 0, count);
else
{
this.bldr.Append(block, 0, count);
brkIx = maxIx;
appendToNext = true;
}
}
internal string GetString(int start, int limit)
{
if (limit <= start)
return "";
if (start >= minIx && limit <= maxIx)
if (limit < brkIx) // String entirely in bldr builder
return bldr.ToString(start - minIx, limit - start);
else if (start >= brkIx) // String entirely in next builder
return next.ToString(start - brkIx, limit - start);
else // Must do a string-concatenation
return
bldr.ToString(start - minIx, brkIx - start) +
next.ToString(0, limit - brkIx);
else
throw new BufferException("String was outside data buffer");
}
internal void Mark(int limit)
{
if (limit > brkIx + 16) // Rotate blocks
{
StringBuilder temp = bldr;
bldr = next;
next = temp;
next.Length = 0;
minIx = brkIx;
brkIx = maxIx;
}
}
}
BufferElement data = new BufferElement();
int bPos; // Postion index in the StringBuilder
BlockReader NextBlk; // Delegate that serves char-arrays;
private string EncodingName
{
get
{
StreamReader rdr = NextBlk.Target as StreamReader;
return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
}
}
public BuildBuffer(Stream stream)
{
FileStream fStrm = (stream as FileStream);
if (fStrm != null) FileName = fStrm.Name;
NextBlk = BlockReaderFactory.Raw(stream);
}
#if (!BYTEMODE)
public BuildBuffer(Stream stream, int fallbackCodePage)
{
FileStream fStrm = (stream as FileStream);
if (fStrm != null) FileName = fStrm.Name;
NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
}
#endif
/// <summary>
/// Marks a conservative lower bound for the buffer,
/// allowing space to be reclaimed. If an application
/// needs to call GetString at arbitrary past locations
/// in the input stream, Mark() is not called.
/// </summary>
public override void Mark() { data.Mark(bPos - 2); }
public override int Pos
{
get { return bPos; }
set { bPos = value; }
}
/// <summary>
/// Read returns the ordinal number of the next char, or
/// EOF (-1) for an end of stream. Note that the next
/// code point may require *two* calls of Read().
/// </summary>
/// <returns></returns>
public override int Read()
{
//
// Characters at positions
// [data.offset, data.offset + data.bldr.Length)
// are available in data.bldr.
//
if (bPos < data.MaxIndex)
{
// ch0 cannot be EOF
return (int)data[bPos++];
}
else // Read from underlying stream
{
// Experimental code, blocks of page size
char[] chrs = new char[4096];
int count = NextBlk(chrs, 0, 4096);
if (count == 0)
return EndOfFile;
else
{
data.Append(chrs, count);
return (int)data[bPos++];
}
}
}
public override string GetString(int begin, int limit)
{
return data.GetString(begin, limit);
}
public override string ToString()
{
return "StringBuilder buffer, encoding: " + this.EncodingName;
}
}
// =============== End ScanBuff-derived classes ==================
internal delegate int BlockReader(char[] block, int index, int number);
// A delegate factory, serving up a delegate that
// reads a block of characters from the underlying
// encoded stream, via a StreamReader object.
//
internal static class BlockReaderFactory
{
public static BlockReader Raw(Stream stream)
{
return delegate(char[] block, int index, int number)
{
byte[] b = new byte[number];
int count = stream.Read(b, 0, number);
int i = 0;
int j = index;
for (; i < count; i++, j++)
block[j] = (char)b[i];
return count;
};
}
#if (!BYTEMODE)
public static BlockReader Get(Stream stream, int fallbackCodePage)
{
Encoding encoding;
int preamble = Preamble(stream);
if (preamble != 0) // There is a valid BOM here!
encoding = Encoding.GetEncoding(preamble);
else if (fallbackCodePage == -1) // Fallback is "raw" bytes
return Raw(stream);
else if (fallbackCodePage != -2) // Anything but "guess"
encoding = Encoding.GetEncoding(fallbackCodePage);
else // This is the "guess" option
{
int guess = new Guesser(stream).GuessCodePage();
stream.Seek(0, SeekOrigin.Begin);
if (guess == -1) // ==> this is a 7-bit file
encoding = Encoding.ASCII;
else if (guess == 65001)
encoding = Encoding.UTF8;
else // ==> use the machine default
encoding = Encoding.Default;
}
StreamReader reader = new StreamReader(stream, encoding);
return reader.Read;
}
static int Preamble(Stream stream)
{
int b0 = stream.ReadByte();
int b1 = stream.ReadByte();
if (b0 == 0xfe && b1 == 0xff)
return 1201; // UTF16BE
if (b0 == 0xff && b1 == 0xfe)
return 1200; // UTF16LE
int b2 = stream.ReadByte();
if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
return 65001; // UTF8
//
// There is no unicode preamble, so we
// return denoter for the machine default.
//
stream.Seek(0, SeekOrigin.Begin);
return 0;
}
#endif // !BYTEMODE
}
#endif // !NOFILES
#endregion Buffer classes
// ==============================================================
// ============ class CodePageHandling =============
// ==============================================================
#if (!NOFILES)
public static class CodePageHandling
{
public static int GetCodePage(string option)
{
string command = option.ToUpperInvariant();
if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
command = command.Substring(9);
try
{
if (command.Equals("RAW"))
return -1;
else if (command.Equals("GUESS"))
return -2;
else if (command.Equals("DEFAULT"))
return 0;
else if (char.IsDigit(command[0]))
return int.Parse(command, CultureInfo.InvariantCulture);
else
{
Encoding enc = Encoding.GetEncoding(command);
return enc.CodePage;
}
}
catch (FormatException)
{
Console.Error.WriteLine(
"Invalid format \"{0}\", using machine default", option);
}
catch (ArgumentException)
{
Console.Error.WriteLine(
"Unknown code page \"{0}\", using machine default", option);
}
return 0;
}
}
#region guesser
#if (!BYTEMODE)
// ==============================================================
// ============ Encoding Guesser =============
// ==============================================================
/// <summary>
/// This class provides a simple finite state automaton that
/// scans the file looking for (1) valid UTF-8 byte patterns,
/// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
/// The method then guesses whether it is UTF-8 or maybe some
/// local machine default encoding. This works well for the
/// various Latin encodings.
/// </summary>
internal class Guesser
{
ScanBuff buffer;
public int GuessCodePage() { return Scan(); }
const int maxAccept = 10;
const int initial = 0;
const int eofNum = 0;
const int goStart = -1;
const int INITIAL = 0;
const int EndToken = 0;
#region user code
/*
* Reads the bytes of a file to determine if it is
* UTF-8 or a single-byte code page file.
*/
public long utfX;
public long uppr;
#endregion user code
int state;
int currentStart = startState[0];
int code;
#region ScannerTables
static int[] startState = new int[] { 11, 0 };
#region CharacterMap
static sbyte[] map = new sbyte[256] {
/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
#endregion
static sbyte[][] nextState = new sbyte[][] {
new sbyte[] {0, 0, 0, 0, 0, 0},
new sbyte[] {-1, -1, 10, -1, -1, -1},
new sbyte[] {-1, -1, -1, -1, -1, -1},
new sbyte[] {-1, -1, 8, -1, -1, -1},
new sbyte[] {-1, -1, 5, -1, -1, -1},
new sbyte[] {-1, -1, 6, -1, -1, -1},
new sbyte[] {-1, -1, 7, -1, -1, -1},
null,
new sbyte[] {-1, -1, 9, -1, -1, -1},
null,
null,
new sbyte[] {-1, 1, 2, 3, 4, 2}
};
[SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
// Reason for suppression: cannot have self-reference in array initializer.
static Guesser()
{
nextState[7] = nextState[2];
nextState[9] = nextState[2];
nextState[10] = nextState[2];
}
int NextState()
{
if (code == ScanBuff.EndOfFile)
return eofNum;
else
return nextState[state][map[code]];
}
#endregion
public Guesser(System.IO.Stream file) { SetSource(file); }
public void SetSource(System.IO.Stream source)
{
this.buffer = new BuildBuffer(source);
code = buffer.Read();
}
int Scan()
{
for (; ; )
{
int next;
state = currentStart;
while ((next = NextState()) == goStart)
code = buffer.Read();
state = next;
code = buffer.Read();
while ((next = NextState()) > eofNum)
{
state = next;
code = buffer.Read();
}
if (state <= maxAccept)
{
#region ActionSwitch
#pragma warning disable 162
switch (state)
{
case eofNum:
switch (currentStart)
{
case 11:
if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
else if (uppr * 10 > utfX) return 0; /* default code page */
else return 65001; /* UTF-8 encoding */
break;
}
return EndToken;
case 1: // Recognized '{Upper128}', Shortest string "\xC0"
case 2: // Recognized '{Upper128}', Shortest string "\x80"
case 3: // Recognized '{Upper128}', Shortest string "\xE0"
case 4: // Recognized '{Upper128}', Shortest string "\xF0"
uppr++;
break;
case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80"
uppr += 2;
break;
case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80"
uppr += 3;
break;
case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80"
utfX += 3;
break;
case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80"
uppr += 2;
break;
case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80"
utfX += 2;
break;
case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80"
utfX++;
break;
default:
break;
}
#pragma warning restore 162
#endregion
}
}
}
} // end class Guesser
#endif // !BYTEMODE
#endregion
#endif // !NOFILES
// End of code copied from embedded resource
}
@@ -1,884 +0,0 @@
// ==============================================================
// <auto-generated>
// This code automatically produced from an embedded resource.
// Do not edit this file, or it will become incompatible with
// the specification from which it was generated.
// </auto-generated>
// ==============================================================
// Gardens Point Parser Generator
// Copyright (c) Wayne Kelly, QUT 2005-2010
// (see accompanying GPPGcopyright.rtf)
using System;
using System.Text;
using System.Globalization;
using System.Collections.Generic;
using System.Runtime.Serialization;
using System.Diagnostics.CodeAnalysis;
namespace QUT.Gppg {
/// <summary>
/// Abstract class for GPPG shift-reduce parsers.
/// Parsers generated by GPPG derive from this base
/// class, overriding the abstract Initialize() and
/// DoAction() methods.
/// </summary>
/// <typeparam name="TValue">Semantic value type</typeparam>
/// <typeparam name="TSpan">Location type</typeparam>
#if EXPORT_GPPG
public abstract class ShiftReduceParser<TValue, TSpan>
#else
internal abstract class ShiftReduceParser<TValue, TSpan>
#endif
where TSpan : IMerge<TSpan>, new() {
private AbstractScanner<TValue, TSpan> scanner;
/// <summary>
/// The abstract scanner for this parser.
/// </summary>
protected AbstractScanner<TValue, TSpan> Scanner {
get { return scanner; }
set { scanner = value; }
}
/// <summary>
/// Constructor for base class
/// </summary>
/// <param name="scanner">Scanner instance for this parser</param>
protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
this.scanner = scanner;
}
// ==============================================================
// TECHNICAL EXPLANATION.
// Why the next two fields are not exposed via properties.
// ==============================================================
// These fields are of the generic parameter types, and are
// frequently instantiated as struct types in derived classes.
// Semantic actions are defined in the derived classes and refer
// to instance fields of these structs. In such cases the code
// "get_CurrentSemanticValue().myField = blah;" will fail since
// the getter pushes the value of the field, not the reference.
// So, in the presence of properties, gppg would need to encode
// such field accesses as ...
// "tmp = get_CurrentSemanticValue(); // Fetch value
// tmp.myField = blah; // update
// set_CurrentSemanticValue(tmp); " // Write update back.
// There is no issue if TValue is restricted to be a ref type.
// The same explanation applies to scanner.yylval.
// ==============================================================
/// <summary>
/// The current value of the "$$" symbolic variable in the parser
/// </summary>
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
protected TValue CurrentSemanticValue;
/// <summary>
/// The current value of the "@$" symbolic variable in the parser
/// </summary>
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
protected TSpan CurrentLocationSpan;
protected int NextToken;
private TSpan LastSpan;
private State FsaState;
private bool recovering;
private int tokensSinceLastError;
private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
/// <summary>
/// The stack of semantic value (YYSTYPE) values.
/// </summary>
protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
/// <summary>
/// The stack of location value (YYLTYPE) varlues.
/// </summary>
protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
private int errorToken;
private int endOfFileToken;
private string[] nonTerminals;
private State[] states;
private Rule[] rules;
/// <summary>
/// Initialization method to allow derived classes
/// to insert the rule list into this base class.
/// </summary>
/// <param name="rules">The array of Rule objects</param>
protected void InitRules( Rule[] rules ) { this.rules = rules; }
/// <summary>
/// Initialization method to allow derived classes to
/// insert the states table into this base class.
/// </summary>
/// <param name="states">The pre-initialized states table</param>
protected void InitStates( State[] states ) { this.states = states; }
/// <summary>
/// OBSOLETE FOR VERSION 1.4.0
/// </summary>
/// <param name="size"></param>
protected void InitStateTable( int size ) { states = new State[size]; }
/// <summary>
/// Initialization method to allow derived classes
/// to insert the special value for the error and EOF tokens.
/// </summary>
/// <param name="err">The error state ordinal</param>
/// <param name="end">The EOF stat ordinal</param>
protected void InitSpecialTokens( int err, int end ) {
errorToken = err;
endOfFileToken = end;
}
/// <summary>
/// Initialization method to allow derived classes to
/// insert the non-terminal symbol names into this base class.
/// </summary>
/// <param name="names">Non-terminal symbol names</param>
protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
#region YYAbort, YYAccept etcetera.
[Serializable]
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
// Reason for FxCop message suppression -
// This exception cannot escape from the local context
private class AcceptException : Exception {
internal AcceptException() { }
protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
}
[Serializable]
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
// Reason for FxCop message suppression -
// This exception cannot escape from the local context
private class AbortException : Exception {
internal AbortException() { }
protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
}
[Serializable]
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
// Reason for FxCop message suppression -
// This exception cannot escape from the local context
private class ErrorException : Exception {
internal ErrorException() { }
protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
}
// The following methods are only called from within
// a semantic action. The thrown exceptions can never
// propagate outside the ShiftReduceParser class in
// which they are nested.
/// <summary>
/// Force parser to terminate, returning "true"
/// </summary>
protected static void YYAccept() { throw new AcceptException(); }
/// <summary>
/// Force parser to terminate, returning "false"
/// </summary>
protected static void YYAbort() { throw new AbortException(); }
/// <summary>
/// Force parser to terminate, returning
/// "false" if error recovery fails.
/// </summary>
protected static void YYError() { throw new ErrorException(); }
/// <summary>
/// Check if parser in error recovery state.
/// </summary>
protected bool YYRecovering { get { return recovering; } }
#endregion
/// <summary>
/// Abstract base method. ShiftReduceParser calls this
/// to initialize the base class data structures. Concrete
/// parser classes must override this method.
/// </summary>
protected abstract void Initialize();
/// <summary>
/// Main entry point of the Shift-Reduce Parser.
/// </summary>
/// <returns>True if parse succeeds, else false for
/// unrecoverable errors</returns>
public bool Parse() {
Initialize(); // allow derived classes to instantiate rules, states and nonTerminals
NextToken = 0;
FsaState = states[0];
StateStack.Push( FsaState );
valueStack.Push( CurrentSemanticValue );
LocationStack.Push( CurrentLocationSpan );
while (true) {
#if TRACE_ACTIONS
Console.Error.WriteLine("Entering state {0} ", FsaState.number);
DisplayStack();
#endif
int action = FsaState.defaultAction;
if (FsaState.ParserTable != null) {
if (NextToken == 0) {
// We save the last token span, so that the location span
// of production right hand sides that begin or end with a
// nullable production will be correct.
LastSpan = scanner.yylloc;
NextToken = scanner.yylex();
#if TRACE_ACTIONS
Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
#endif
}
#if TRACE_ACTIONS
else
Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
#endif
if (FsaState.ParserTable.ContainsKey( NextToken ))
action = FsaState.ParserTable[NextToken];
}
if (action > 0) // shift
{
Shift( action );
}
else if (action < 0) // reduce
{
try {
Reduce( -action );
if (action == -1) // accept
return true;
}
catch (AbortException) {
return false;
}
catch (AcceptException) {
return true;
}
catch (ErrorException) {
if (!ErrorRecovery())
return false;
else
throw; // Rethrow x, preserving information.
}
}
else if (action == 0 && !ErrorRecovery()) // error
return false;
}
}
private void Shift( int stateIndex ) {
#if TRACE_ACTIONS
Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
#endif
FsaState = states[stateIndex];
valueStack.Push( scanner.yylval );
StateStack.Push( FsaState );
LocationStack.Push( scanner.yylloc );
if (recovering) {
if (NextToken != errorToken)
tokensSinceLastError++;
if (tokensSinceLastError > 5)
recovering = false;
}
if (NextToken != endOfFileToken)
NextToken = 0;
}
private void Reduce( int ruleNumber ) {
#if TRACE_ACTIONS
DisplayRule(ruleNumber);
#endif
Rule rule = rules[ruleNumber];
int rhLen = rule.RightHandSide.Length;
//
// Default actions for unit productions.
//
if (rhLen == 1) {
CurrentSemanticValue = valueStack.TopElement(); // Default action: $$ = $1;
CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
}
else if (rhLen == 0) {
// Create a new blank value.
// Explicit semantic action may mutate this value
CurrentSemanticValue = default( TValue );
// The location span for an empty production will start with the
// beginning of the next lexeme, and end with the finish of the
// previous lexeme. This gives the correct behaviour when this
// nonsense value is used in later Merge operations.
CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
scanner.yylloc.Merge( LastSpan ) :
default( TSpan ));
}
else {
// Default action: $$ = $1;
CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
// Default action "@$ = @1.Merge(@N)" for location info.
TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
TSpan atN = LocationStack[LocationStack.Depth - 1];
CurrentLocationSpan =
((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
}
DoAction( ruleNumber );
for (int i = 0; i < rule.RightHandSide.Length; i++) {
StateStack.Pop();
valueStack.Pop();
LocationStack.Pop();
}
FsaState = StateStack.TopElement();
if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
FsaState = states[FsaState.Goto[rule.LeftHandSide]];
StateStack.Push( FsaState );
valueStack.Push( CurrentSemanticValue );
LocationStack.Push( CurrentLocationSpan );
}
/// <summary>
/// Execute the selected action from array.
/// Must be overriden in derived classes.
/// </summary>
/// <param name="actionNumber">Index of the action to perform</param>
protected abstract void DoAction( int actionNumber );
private bool ErrorRecovery() {
bool discard;
if (!recovering) // if not recovering from previous error
ReportError();
if (!FindErrorRecoveryState())
return false;
//
// The interim fix for the "looping in error recovery"
// artifact involved moving the setting of the recovering
// bool until after invalid tokens have been discarded.
//
ShiftErrorToken();
discard = DiscardInvalidTokens();
recovering = true;
tokensSinceLastError = 0;
return discard;
}
private void ReportError() {
StringBuilder errorMsg = new StringBuilder();
errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
if (FsaState.ParserTable.Count < 7) {
bool first = true;
foreach (int terminal in FsaState.ParserTable.Keys) {
if (first)
errorMsg.Append( ", expecting " );
else
errorMsg.Append( ", or " );
errorMsg.Append( TerminalToString( terminal ) );
first = false;
}
}
scanner.yyerror( errorMsg.ToString() );
}
private void ShiftErrorToken() {
int old_next = NextToken;
NextToken = errorToken;
Shift( FsaState.ParserTable[NextToken] );
#if TRACE_ACTIONS
Console.Error.WriteLine("Entering state {0} ", FsaState.number);
#endif
NextToken = old_next;
}
private bool FindErrorRecoveryState() {
while (true) // pop states until one found that accepts error token
{
if (FsaState.ParserTable != null &&
FsaState.ParserTable.ContainsKey( errorToken ) &&
FsaState.ParserTable[errorToken] > 0) // shift
return true;
#if TRACE_ACTIONS
Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
#endif
StateStack.Pop();
valueStack.Pop();
LocationStack.Pop();
#if TRACE_ACTIONS
DisplayStack();
#endif
if (StateStack.IsEmpty()) {
#if TRACE_ACTIONS
Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
#endif
return false;
}
else
FsaState = StateStack.TopElement();
}
}
private bool DiscardInvalidTokens() {
int action = FsaState.defaultAction;
if (FsaState.ParserTable != null) {
// Discard tokens until find one that works ...
while (true) {
if (NextToken == 0) {
#if TRACE_ACTIONS
Console.Error.Write("Reading a token: ");
#endif
NextToken = scanner.yylex();
}
#if TRACE_ACTIONS
Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
#endif
if (NextToken == endOfFileToken)
return false;
if (FsaState.ParserTable.ContainsKey( NextToken ))
action = FsaState.ParserTable[NextToken];
if (action != 0)
return true;
else {
#if TRACE_ACTIONS
Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
#endif
NextToken = 0;
}
}
}
else if (recovering && tokensSinceLastError == 0) {
//
// Boolean recovering is not set until after the first
// error token has been shifted. Thus if we get back
// here with recovering set and no tokens read we are
// looping on the same error recovery action. This
// happens if current_state.ParserTable is null because
// the state has an LR(0) reduction, but not all
// lookahead tokens are valid. This only occurs for
// error productions that *end* on "error".
//
// This action discards tokens one at a time until
// the looping stops. Another attack would be to always
// use the LALR(1) table if a production ends on "error"
//
#if TRACE_ACTIONS
Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
#endif
if (NextToken == endOfFileToken)
return false;
NextToken = 0;
return true;
}
else
return true;
}
/// <summary>
/// Traditional YACC method. Discards the next input token.
/// </summary>
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
protected void yyclearin() { NextToken = 0; }
/// <summary>
/// Tradional YACC method. Clear the "recovering" flag.
/// </summary>
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
protected void yyerrok() { recovering = false; }
/// <summary>
/// OBSOLETE FOR VERSION 1.4.0
/// Method used by derived types to insert new
/// state instances in the "states" array.
/// </summary>
/// <param name="stateNumber">index of the state</param>
/// <param name="state">data for the state</param>
protected void AddState( int stateNumber, State state ) {
states[stateNumber] = state;
state.number = stateNumber;
}
private void DisplayStack() {
Console.Error.Write( "State stack is now:" );
for (int i = 0; i < StateStack.Depth; i++)
Console.Error.Write( " {0}", StateStack[i].number );
Console.Error.WriteLine();
}
private void DisplayRule( int ruleNumber ) {
Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
DisplayProduction( rules[ruleNumber] );
}
private void DisplayProduction( Rule rule ) {
if (rule.RightHandSide.Length == 0)
Console.Error.Write( "/* empty */ " );
else
foreach (int symbol in rule.RightHandSide)
Console.Error.Write( "{0} ", SymbolToString( symbol ) );
Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
}
/// <summary>
/// Abstract state class naming terminal symbols.
/// This is overridden by derived classes with the
/// name (or alias) to be used in error messages.
/// </summary>
/// <param name="terminal">The terminal ordinal</param>
/// <returns></returns>
protected abstract string TerminalToString( int terminal );
private string SymbolToString( int symbol ) {
if (symbol < 0)
return nonTerminals[-symbol - 1];
else
return TerminalToString( symbol );
}
/// <summary>
/// Return text representation of argument character
/// </summary>
/// <param name="input">The character to convert</param>
/// <returns>String representation of the character</returns>
protected static string CharToString( char input ) {
switch (input) {
case '\a': return @"'\a'";
case '\b': return @"'\b'";
case '\f': return @"'\f'";
case '\n': return @"'\n'";
case '\r': return @"'\r'";
case '\t': return @"'\t'";
case '\v': return @"'\v'";
case '\0': return @"'\0'";
default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
}
}
}
/// <summary>
/// Classes implementing this interface must supply a
/// method that merges two location objects to return
/// a new object of the same type.
/// GPPG-generated parsers have the default location
/// action equivalent to "@$ = @1.Merge(@N);" where N
/// is the right-hand-side length of the production.
/// </summary>
/// <typeparam name="TSpan">The Location type</typeparam>
#if EXPORT_GPPG
public interface IMerge<TSpan>
#else
internal interface IMerge<TSpan>
#endif
{
/// <summary>
/// Interface method that creates a location object from
/// the current and last object. Typically used to create
/// a location object extending from the start of the @1
/// object to the end of the @N object.
/// </summary>
/// <param name="last">The lexically last object to merge</param>
/// <returns>The merged location object</returns>
TSpan Merge( TSpan last );
}
/// <summary>
/// This is the default class that carries location
/// information from the scanner to the parser.
/// If you don't declare "%YYLTYPE Foo" the parser
/// will expect to deal with this type.
/// </summary>
#if EXPORT_GPPG
public class LexLocation : IMerge<LexLocation>
#else
[SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
internal class LexLocation : IMerge<LexLocation>
#endif
{
private int startLine; // start line
private int startColumn; // start column
private int endLine; // end line
private int endColumn; // end column
/// <summary>
/// The line at which the text span starts.
/// </summary>
public int StartLine { get { return startLine; } }
/// <summary>
/// The column at which the text span starts.
/// </summary>
public int StartColumn { get { return startColumn; } }
/// <summary>
/// The line on which the text span ends.
/// </summary>
public int EndLine { get { return endLine; } }
/// <summary>
/// The column of the first character
/// beyond the end of the text span.
/// </summary>
public int EndColumn { get { return endColumn; } }
/// <summary>
/// Default no-arg constructor.
/// </summary>
public LexLocation() { }
/// <summary>
/// Constructor for text-span with given start and end.
/// </summary>
/// <param name="sl">start line</param>
/// <param name="sc">start column</param>
/// <param name="el">end line </param>
/// <param name="ec">end column</param>
public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
/// <summary>
/// Create a text location which spans from the
/// start of "this" to the end of the argument "last"
/// </summary>
/// <param name="last">The last location in the result span</param>
/// <returns>The merged span</returns>
public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
}
/// <summary>
/// Abstract scanner class that GPPG expects its scanners to
/// extend.
/// </summary>
/// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
/// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
#if EXPORT_GPPG
public abstract class AbstractScanner<TValue, TSpan>
#else
internal abstract class AbstractScanner<TValue, TSpan>
#endif
where TSpan : IMerge<TSpan> {
/// <summary>
/// Lexical value optionally set by the scanner. The value
/// is of the %YYSTYPE type declared in the parser spec.
/// </summary>
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
// A field must be declared for this value of parametric type,
// since it may be instantiated by a value struct. If it were
// implemented as a property, machine generated code in derived
// types would not be able to select on the returned value.
#pragma warning disable 649
public TValue yylval; // Lexical value: set by scanner
#pragma warning restore 649
/// <summary>
/// Current scanner location property. The value is of the
/// type declared by %YYLTYPE in the parser specification.
/// </summary>
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
public virtual TSpan yylloc {
get { return default( TSpan ); } // Empty implementation allowing
set { /* skip */ } // yylloc to be ignored entirely.
}
/// <summary>
/// Main call point for LEX-like scanners. Returns an int
/// corresponding to the token recognized by the scanner.
/// </summary>
/// <returns>An int corresponding to the token</returns>
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
public abstract int yylex();
/// <summary>
/// Traditional error reporting provided by LEX-like scanners
/// to their YACC-like clients.
/// </summary>
/// <param name="format">Message format string</param>
/// <param name="args">Optional array of args</param>
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
public virtual void yyerror( string format, params object[] args ) { }
}
/// <summary>
/// Encapsulated state for the parser.
/// Opaque to users, visible to the tool-generated code.
/// </summary>
#if EXPORT_GPPG
public class State
{
/// <summary>
/// The number of states in the automaton.
/// </summary>
public int number;
#else
internal class State {
/// <summary>
/// The index of this state in the states array.
/// </summary>
internal int number;
#endif
internal Dictionary<int, int> ParserTable; // Terminal -> ParseAction
internal Dictionary<int, int> Goto; // NonTerminal -> State;
internal int defaultAction; // = 0; // ParseAction
/// <summary>
/// State transition data for this state. Pairs of elements of the
/// goto array associate symbol ordinals with next state indices.
/// The actions array is passed to another constructor.
/// </summary>
/// <param name="actions">The action list</param>c
/// <param name="goToList">Next state data</param>
public State( int[] actions, int[] goToList )
: this( actions ) {
Goto = new Dictionary<int, int>();
for (int i = 0; i < goToList.Length; i += 2)
Goto.Add( goToList[i], goToList[i + 1] );
}
/// <summary>
/// Action data for this state. Pairs of elements of the
/// action array associate action ordinals with each of
/// those symbols that have actions in the current state.
/// </summary>
/// <param name="actions">The action array</param>
public State( int[] actions ) {
ParserTable = new Dictionary<int, int>();
for (int i = 0; i < actions.Length; i += 2)
ParserTable.Add( actions[i], actions[i + 1] );
}
/// <summary>
/// Set the default action for this state.
/// </summary>
/// <param name="defaultAction">Ordinal of the default action</param>
public State( int defaultAction ) {
this.defaultAction = defaultAction;
}
/// <summary>
/// Set the default action and the state transition table.
/// </summary>
/// <param name="defaultAction">The default action</param>
/// <param name="goToList">Transitions from this state</param>
public State( int defaultAction, int[] goToList )
: this( defaultAction ) {
Goto = new Dictionary<int, int>();
for (int i = 0; i < goToList.Length; i += 2)
Goto.Add( goToList[i], goToList[i + 1] );
}
}
/// <summary>
/// Rule representation at runtime.
/// </summary>
#if EXPORT_GPPG
public class Rule
#else
internal class Rule
#endif
{
internal int LeftHandSide; // symbol
internal int[] RightHandSide; // symbols
/// <summary>
/// Rule constructor. This holds the ordinal of
/// the left hand side symbol, and the list of
/// right hand side symbols, in lexical order.
/// </summary>
/// <param name="left">The LHS non-terminal</param>
/// <param name="right">The RHS symbols, in lexical order</param>
public Rule( int left, int[] right ) {
this.LeftHandSide = left;
this.RightHandSide = right;
}
}
/// <summary>
/// Stack utility for the shift-reduce parser.
/// GPPG parsers have three instances:
/// (1) The parser state stack, T = QUT.Gppg.State,
/// (2) The semantic value stack, T = TValue,
/// (3) The location stack, T = TSpan.
/// </summary>
/// <typeparam name="T"></typeparam>
#if EXPORT_GPPG
public class PushdownPrefixState<T>
#else
internal class PushdownPrefixState<T>
#endif
{
// Note that we cannot use the BCL Stack<T> class
// here as derived types need to index into stacks.
//
private T[] array = new T[8];
private int tos = 0;
/// <summary>
/// Indexer for values of the stack below the top.
/// </summary>
/// <param name="index">index of the element, starting from the bottom</param>
/// <returns>the selected element</returns>
public T this[int index] { get { return array[index]; } }
/// <summary>
/// The current depth of the stack.
/// </summary>
public int Depth { get { return tos; } }
internal void Push( T value ) {
if (tos >= array.Length) {
T[] newarray = new T[array.Length * 2];
System.Array.Copy( array, newarray, tos );
array = newarray;
}
array[tos++] = value;
}
internal T Pop() {
T rslt = array[--tos];
array[tos] = default( T );
return rslt;
}
internal T TopElement() { return array[tos - 1]; }
internal bool IsEmpty() { return tos == 0; }
}
}
@@ -1,4 +0,0 @@
global using System;
global using System.Linq;
global using System.Collections.Generic;
global using System.Collections.Immutable;
@@ -1,24 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="YaccLexTools" Version="1.2.2" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
<ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
</ItemGroup>
<ItemGroup Label="ColumnSortExpression">
<Scanner Include="ColumnSortExpressionTokens.lex" />
<Parser Include="ColumnSortExpressionGrammar.y" />
</ItemGroup>
</Project>
@@ -1,11 +0,0 @@
namespace Bts.ColumnSortExpressions;
public class ColumnSortExpression {
public ColumnSortExpression(string name, bool isAscending = true) {
Name = name;
IsAscending = isAscending;
}
public string Name { get; }
public bool IsAscending { get; }
}
@@ -1,29 +0,0 @@
%namespace Bts.ColumnSortExpressions
%partial
%visibility internal
%tokentype ColumnSortExpressionToken
%parsertype ColumnSortExpressionParser
%union {
public string Identifier;
public ColumnSortExpression Column;
public ImmutableList<ColumnSortExpression> Columns;
}
%token ASCENDING, DESCENDING, IDENTIFIER
%type <Column> Column
%type <Columns> Columns
%%
Columns :
Column { $$ = ImmutableList.CreateRange(new[] { $1 }); }
| Columns Column { $$ = $1.Add($2); }
;
Column :
IDENTIFIER { $$ = new($1.Identifier, true); }
| ASCENDING IDENTIFIER { $$ = new($2.Identifier, true); }
| DESCENDING IDENTIFIER { $$ = new($2.Identifier, false); }
;
@@ -1,18 +0,0 @@
namespace Bts.ColumnSortExpressions;
using System.IO;
using System.Text;
public class ColumnSortExpressionInterpreter {
public static IList<ColumnSortExpression> Evaluate(Stream stream) {
var parser = new ColumnSortExpressionParser();
var res = parser.Parse(stream);
return res?.Columns ?? ImmutableList<ColumnSortExpression>.Empty.Add(
res?.Column ?? throw new Exception("Failed to parse column(s) expression"));
}
public static IList<ColumnSortExpression> Evaluate(string str) {
using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
return Evaluate(ms);
}
}
@@ -1,18 +0,0 @@
namespace Bts.ColumnSortExpressions;
using System.Text;
partial class ColumnSortExpressionParser {
public ColumnSortExpressionParser() : base(null) { }
public ValueType? Parse(string s) {
using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
return Parse(ms);
}
public ValueType? Parse(Stream stream) {
Scanner = new ColumnSortExpressionScanner(stream);
Parse();
return CurrentSemanticValue;
}
}
@@ -1,9 +0,0 @@
namespace Bts.ColumnSortExpressions;
partial class ColumnSortExpressionScanner {
public override void yyerror(string format, params object[] args) {
base.yyerror(format, args);
Console.WriteLine(format, args);
Console.WriteLine();
}
}
@@ -1,30 +0,0 @@
%namespace Bts.ColumnSortExpressions
%scannertype ColumnSortExpressionScanner
%visibility internal
%tokentype ColumnSortExpressionToken
%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers
/* Literal Constants */
Letter [a-zA-Z]
Identifier {Letter}({Letter}*[0-9]*_*)*
Ascending +
Descending -
%%
{Ascending} {
return (int)ColumnSortExpressionToken.ASCENDING;
}
{Descending} {
return (int)ColumnSortExpressionToken.DESCENDING;
}
{Identifier} {
yylval.Identifier = yytext;
return (int)ColumnSortExpressionToken.IDENTIFIER;
}
%%
@@ -1,720 +0,0 @@
// ==============================================================
// <auto-generated>
// This code automatically produced from an embedded resource.
// Do not edit this file, or it will become incompatible with
// the specification from which it was generated.
// </auto-generated>
// ==============================================================
using System;
using System.IO;
using System.Text;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Runtime.Serialization;
using System.Globalization;
namespace QUT.GplexBuffers
{
// Code copied from GPLEX embedded resource
[Serializable]
internal class BufferException : Exception
{
public BufferException() { }
public BufferException(string message) : base(message) { }
public BufferException(string message, Exception innerException)
: base(message, innerException) { }
protected BufferException(SerializationInfo info, StreamingContext context)
: base(info, context) { }
}
internal abstract class ScanBuff
{
private string fileNm;
public const int EndOfFile = -1;
public const int UnicodeReplacementChar = 0xFFFD;
public bool IsFile { get { return (fileNm != null); } }
public string FileName { get { return fileNm; } set { fileNm = value; } }
public abstract int Pos { get; set; }
public abstract int Read();
public virtual void Mark() { }
public abstract string GetString(int begin, int limit);
public static ScanBuff GetBuffer(string source)
{
return new StringBuffer(source);
}
public static ScanBuff GetBuffer(IList<string> source)
{
return new LineBuffer(source);
}
#if (!NOFILES)
public static ScanBuff GetBuffer(Stream source)
{
return new BuildBuffer(source);
}
#if (!BYTEMODE)
public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
{
return new BuildBuffer(source, fallbackCodePage);
}
#endif // !BYTEMODE
#endif // !NOFILES
}
#region Buffer classes
// ==============================================================
// ===== Definitions for various ScanBuff derived classes ====
// ==============================================================
// =============== String input ================
// ==============================================================
/// <summary>
/// This class reads characters from a single string as
/// required, for example, by Visual Studio language services
/// </summary>
sealed class StringBuffer : ScanBuff
{
string str; // input buffer
int bPos; // current position in buffer
int sLen;
public StringBuffer(string source)
{
this.str = source;
this.sLen = source.Length;
this.FileName = null;
}
public override int Read()
{
if (bPos < sLen) return str[bPos++];
else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line
else { bPos++; return EndOfFile; } // two strikes and you're out!
}
public override string GetString(int begin, int limit)
{
// "limit" can be greater than sLen with the BABEL
// option set. Read returns a "virtual" EOL if
// an attempt is made to read past the end of the
// string buffer. Without the guard any attempt
// to fetch yytext for a token that includes the
// EOL will throw an index exception.
if (limit > sLen) limit = sLen;
if (limit <= begin) return "";
else return str.Substring(begin, limit - begin);
}
public override int Pos
{
get { return bPos; }
set { bPos = value; }
}
public override string ToString() { return "StringBuffer"; }
}
// ==============================================================
// The LineBuff class contributed by Nigel Horspool,
// nigelh@cs.uvic.cs
// ==============================================================
sealed class LineBuffer : ScanBuff
{
IList<string> line; // list of source lines from a file
int numLines; // number of strings in line list
string curLine; // current line in that list
int cLine; // index of current line in the list
int curLen; // length of current line
int curLineStart; // position of line start in whole file
int curLineEnd; // position of line end in whole file
int maxPos; // max position ever visited in whole file
int cPos; // ordinal number of code in source
// Constructed from a list of strings, one per source line.
// The lines have had trailing '\n' characters removed.
public LineBuffer(IList<string> lineList)
{
line = lineList;
numLines = line.Count;
cPos = curLineStart = 0;
curLine = (numLines > 0 ? line[0] : "");
maxPos = curLineEnd = curLen = curLine.Length;
cLine = 1;
FileName = null;
}
public override int Read()
{
if (cPos < curLineEnd)
return curLine[cPos++ - curLineStart];
if (cPos++ == curLineEnd)
return '\n';
if (cLine >= numLines)
return EndOfFile;
curLine = line[cLine];
curLen = curLine.Length;
curLineStart = curLineEnd + 1;
curLineEnd = curLineStart + curLen;
if (curLineEnd > maxPos)
maxPos = curLineEnd;
cLine++;
return curLen > 0 ? curLine[0] : '\n';
}
// To speed up searches for the line containing a position
private int cachedPosition;
private int cachedIxdex;
private int cachedLineStart;
// Given a position pos within the entire source, the results are
// ix -- the index of the containing line
// lstart -- the position of the first character on that line
private void findIndex(int pos, out int ix, out int lstart)
{
if (pos >= cachedPosition)
{
ix = cachedIxdex; lstart = cachedLineStart;
}
else
{
ix = lstart = 0;
}
while (ix < numLines)
{
int len = line[ix].Length + 1;
if (pos < lstart + len) break;
lstart += len;
ix++;
}
cachedPosition = pos;
cachedIxdex = ix;
cachedLineStart = lstart;
}
public override string GetString(int begin, int limit)
{
if (begin >= maxPos || limit <= begin) return "";
int endIx, begIx, endLineStart, begLineStart;
findIndex(begin, out begIx, out begLineStart);
int begCol = begin - begLineStart;
findIndex(limit, out endIx, out endLineStart);
int endCol = limit - endLineStart;
string s = line[begIx];
if (begIx == endIx)
{
// the usual case, substring all on one line
return (endCol <= s.Length) ?
s.Substring(begCol, endCol - begCol)
: s.Substring(begCol) + "\n";
}
// the string spans multiple lines, yuk!
StringBuilder sb = new StringBuilder();
if (begCol < s.Length)
sb.Append(s.Substring(begCol));
for (; ; )
{
sb.Append("\n");
s = line[++begIx];
if (begIx >= endIx) break;
sb.Append(s);
}
if (endCol <= s.Length)
{
sb.Append(s.Substring(0, endCol));
}
else
{
sb.Append(s);
sb.Append("\n");
}
return sb.ToString();
}
public override int Pos
{
get { return cPos; }
set
{
cPos = value;
findIndex(cPos, out cLine, out curLineStart);
// cLine should be the *next* line after curLine.
curLine = (cLine < numLines ? line[cLine++] : "");
curLineEnd = curLineStart + curLine.Length;
}
}
public override string ToString() { return "LineBuffer"; }
}
#if (!NOFILES)
// ==============================================================
// ===== class BuildBuff : for unicode text files ========
// ==============================================================
class BuildBuffer : ScanBuff
{
// Double buffer for char stream.
class BufferElement
{
StringBuilder bldr = new StringBuilder();
StringBuilder next = new StringBuilder();
int minIx;
int maxIx;
int brkIx;
bool appendToNext;
internal BufferElement() { }
internal int MaxIndex { get { return maxIx; } }
// internal int MinIndex { get { return minIx; } }
internal char this[int index]
{
get
{
if (index < minIx || index >= maxIx)
throw new BufferException("Index was outside data buffer");
else if (index < brkIx)
return bldr[index - minIx];
else
return next[index - brkIx];
}
}
internal void Append(char[] block, int count)
{
maxIx += count;
if (appendToNext)
this.next.Append(block, 0, count);
else
{
this.bldr.Append(block, 0, count);
brkIx = maxIx;
appendToNext = true;
}
}
internal string GetString(int start, int limit)
{
if (limit <= start)
return "";
if (start >= minIx && limit <= maxIx)
if (limit < brkIx) // String entirely in bldr builder
return bldr.ToString(start - minIx, limit - start);
else if (start >= brkIx) // String entirely in next builder
return next.ToString(start - brkIx, limit - start);
else // Must do a string-concatenation
return
bldr.ToString(start - minIx, brkIx - start) +
next.ToString(0, limit - brkIx);
else
throw new BufferException("String was outside data buffer");
}
internal void Mark(int limit)
{
if (limit > brkIx + 16) // Rotate blocks
{
StringBuilder temp = bldr;
bldr = next;
next = temp;
next.Length = 0;
minIx = brkIx;
brkIx = maxIx;
}
}
}
BufferElement data = new BufferElement();
int bPos; // Postion index in the StringBuilder
BlockReader NextBlk; // Delegate that serves char-arrays;
private string EncodingName
{
get
{
StreamReader rdr = NextBlk.Target as StreamReader;
return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
}
}
public BuildBuffer(Stream stream)
{
FileStream fStrm = (stream as FileStream);
if (fStrm != null) FileName = fStrm.Name;
NextBlk = BlockReaderFactory.Raw(stream);
}
#if (!BYTEMODE)
public BuildBuffer(Stream stream, int fallbackCodePage)
{
FileStream fStrm = (stream as FileStream);
if (fStrm != null) FileName = fStrm.Name;
NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
}
#endif
/// <summary>
/// Marks a conservative lower bound for the buffer,
/// allowing space to be reclaimed. If an application
/// needs to call GetString at arbitrary past locations
/// in the input stream, Mark() is not called.
/// </summary>
public override void Mark() { data.Mark(bPos - 2); }
public override int Pos
{
get { return bPos; }
set { bPos = value; }
}
/// <summary>
/// Read returns the ordinal number of the next char, or
/// EOF (-1) for an end of stream. Note that the next
/// code point may require *two* calls of Read().
/// </summary>
/// <returns></returns>
public override int Read()
{
//
// Characters at positions
// [data.offset, data.offset + data.bldr.Length)
// are available in data.bldr.
//
if (bPos < data.MaxIndex)
{
// ch0 cannot be EOF
return (int)data[bPos++];
}
else // Read from underlying stream
{
// Experimental code, blocks of page size
char[] chrs = new char[4096];
int count = NextBlk(chrs, 0, 4096);
if (count == 0)
return EndOfFile;
else
{
data.Append(chrs, count);
return (int)data[bPos++];
}
}
}
public override string GetString(int begin, int limit)
{
return data.GetString(begin, limit);
}
public override string ToString()
{
return "StringBuilder buffer, encoding: " + this.EncodingName;
}
}
// =============== End ScanBuff-derived classes ==================
internal delegate int BlockReader(char[] block, int index, int number);
// A delegate factory, serving up a delegate that
// reads a block of characters from the underlying
// encoded stream, via a StreamReader object.
//
internal static class BlockReaderFactory
{
public static BlockReader Raw(Stream stream)
{
return delegate(char[] block, int index, int number)
{
byte[] b = new byte[number];
int count = stream.Read(b, 0, number);
int i = 0;
int j = index;
for (; i < count; i++, j++)
block[j] = (char)b[i];
return count;
};
}
#if (!BYTEMODE)
public static BlockReader Get(Stream stream, int fallbackCodePage)
{
Encoding encoding;
int preamble = Preamble(stream);
if (preamble != 0) // There is a valid BOM here!
encoding = Encoding.GetEncoding(preamble);
else if (fallbackCodePage == -1) // Fallback is "raw" bytes
return Raw(stream);
else if (fallbackCodePage != -2) // Anything but "guess"
encoding = Encoding.GetEncoding(fallbackCodePage);
else // This is the "guess" option
{
int guess = new Guesser(stream).GuessCodePage();
stream.Seek(0, SeekOrigin.Begin);
if (guess == -1) // ==> this is a 7-bit file
encoding = Encoding.ASCII;
else if (guess == 65001)
encoding = Encoding.UTF8;
else // ==> use the machine default
encoding = Encoding.Default;
}
StreamReader reader = new StreamReader(stream, encoding);
return reader.Read;
}
static int Preamble(Stream stream)
{
int b0 = stream.ReadByte();
int b1 = stream.ReadByte();
if (b0 == 0xfe && b1 == 0xff)
return 1201; // UTF16BE
if (b0 == 0xff && b1 == 0xfe)
return 1200; // UTF16LE
int b2 = stream.ReadByte();
if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
return 65001; // UTF8
//
// There is no unicode preamble, so we
// return denoter for the machine default.
//
stream.Seek(0, SeekOrigin.Begin);
return 0;
}
#endif // !BYTEMODE
}
#endif // !NOFILES
#endregion Buffer classes
// ==============================================================
// ============ class CodePageHandling =============
// ==============================================================
#if (!NOFILES)
public static class CodePageHandling
{
public static int GetCodePage(string option)
{
string command = option.ToUpperInvariant();
if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
command = command.Substring(9);
try
{
if (command.Equals("RAW"))
return -1;
else if (command.Equals("GUESS"))
return -2;
else if (command.Equals("DEFAULT"))
return 0;
else if (char.IsDigit(command[0]))
return int.Parse(command, CultureInfo.InvariantCulture);
else
{
Encoding enc = Encoding.GetEncoding(command);
return enc.CodePage;
}
}
catch (FormatException)
{
Console.Error.WriteLine(
"Invalid format \"{0}\", using machine default", option);
}
catch (ArgumentException)
{
Console.Error.WriteLine(
"Unknown code page \"{0}\", using machine default", option);
}
return 0;
}
}
#region guesser
#if (!BYTEMODE)
// ==============================================================
// ============ Encoding Guesser =============
// ==============================================================
/// <summary>
/// This class provides a simple finite state automaton that
/// scans the file looking for (1) valid UTF-8 byte patterns,
/// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
/// The method then guesses whether it is UTF-8 or maybe some
/// local machine default encoding. This works well for the
/// various Latin encodings.
/// </summary>
internal class Guesser
{
ScanBuff buffer;
public int GuessCodePage() { return Scan(); }
const int maxAccept = 10;
const int initial = 0;
const int eofNum = 0;
const int goStart = -1;
const int INITIAL = 0;
const int EndToken = 0;
#region user code
/*
* Reads the bytes of a file to determine if it is
* UTF-8 or a single-byte code page file.
*/
public long utfX;
public long uppr;
#endregion user code
int state;
int currentStart = startState[0];
int code;
#region ScannerTables
static int[] startState = new int[] { 11, 0 };
#region CharacterMap
static sbyte[] map = new sbyte[256] {
/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
#endregion
static sbyte[][] nextState = new sbyte[][] {
new sbyte[] {0, 0, 0, 0, 0, 0},
new sbyte[] {-1, -1, 10, -1, -1, -1},
new sbyte[] {-1, -1, -1, -1, -1, -1},
new sbyte[] {-1, -1, 8, -1, -1, -1},
new sbyte[] {-1, -1, 5, -1, -1, -1},
new sbyte[] {-1, -1, 6, -1, -1, -1},
new sbyte[] {-1, -1, 7, -1, -1, -1},
null,
new sbyte[] {-1, -1, 9, -1, -1, -1},
null,
null,
new sbyte[] {-1, 1, 2, 3, 4, 2}
};
[SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
// Reason for suppression: cannot have self-reference in array initializer.
static Guesser()
{
nextState[7] = nextState[2];
nextState[9] = nextState[2];
nextState[10] = nextState[2];
}
int NextState()
{
if (code == ScanBuff.EndOfFile)
return eofNum;
else
return nextState[state][map[code]];
}
#endregion
public Guesser(System.IO.Stream file) { SetSource(file); }
public void SetSource(System.IO.Stream source)
{
this.buffer = new BuildBuffer(source);
code = buffer.Read();
}
int Scan()
{
for (; ; )
{
int next;
state = currentStart;
while ((next = NextState()) == goStart)
code = buffer.Read();
state = next;
code = buffer.Read();
while ((next = NextState()) > eofNum)
{
state = next;
code = buffer.Read();
}
if (state <= maxAccept)
{
#region ActionSwitch
#pragma warning disable 162
switch (state)
{
case eofNum:
switch (currentStart)
{
case 11:
if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
else if (uppr * 10 > utfX) return 0; /* default code page */
else return 65001; /* UTF-8 encoding */
break;
}
return EndToken;
case 1: // Recognized '{Upper128}', Shortest string "\xC0"
case 2: // Recognized '{Upper128}', Shortest string "\x80"
case 3: // Recognized '{Upper128}', Shortest string "\xE0"
case 4: // Recognized '{Upper128}', Shortest string "\xF0"
uppr++;
break;
case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80"
uppr += 2;
break;
case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80"
uppr += 3;
break;
case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80"
utfX += 3;
break;
case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80"
uppr += 2;
break;
case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80"
utfX += 2;
break;
case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80"
utfX++;
break;
default:
break;
}
#pragma warning restore 162
#endregion
}
}
}
} // end class Guesser
#endif // !BYTEMODE
#endregion
#endif // !NOFILES
// End of code copied from embedded resource
}
@@ -1,884 +0,0 @@
// ==============================================================
// <auto-generated>
// This code automatically produced from an embedded resource.
// Do not edit this file, or it will become incompatible with
// the specification from which it was generated.
// </auto-generated>
// ==============================================================
// Gardens Point Parser Generator
// Copyright (c) Wayne Kelly, QUT 2005-2010
// (see accompanying GPPGcopyright.rtf)
using System;
using System.Text;
using System.Globalization;
using System.Collections.Generic;
using System.Runtime.Serialization;
using System.Diagnostics.CodeAnalysis;
namespace QUT.Gppg {
/// <summary>
/// Abstract class for GPPG shift-reduce parsers.
/// Parsers generated by GPPG derive from this base
/// class, overriding the abstract Initialize() and
/// DoAction() methods.
/// </summary>
/// <typeparam name="TValue">Semantic value type</typeparam>
/// <typeparam name="TSpan">Location type</typeparam>
#if EXPORT_GPPG
public abstract class ShiftReduceParser<TValue, TSpan>
#else
internal abstract class ShiftReduceParser<TValue, TSpan>
#endif
where TSpan : IMerge<TSpan>, new() {
private AbstractScanner<TValue, TSpan> scanner;
/// <summary>
/// The abstract scanner for this parser.
/// </summary>
protected AbstractScanner<TValue, TSpan> Scanner {
get { return scanner; }
set { scanner = value; }
}
/// <summary>
/// Constructor for base class
/// </summary>
/// <param name="scanner">Scanner instance for this parser</param>
protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
this.scanner = scanner;
}
// ==============================================================
// TECHNICAL EXPLANATION.
// Why the next two fields are not exposed via properties.
// ==============================================================
// These fields are of the generic parameter types, and are
// frequently instantiated as struct types in derived classes.
// Semantic actions are defined in the derived classes and refer
// to instance fields of these structs. In such cases the code
// "get_CurrentSemanticValue().myField = blah;" will fail since
// the getter pushes the value of the field, not the reference.
// So, in the presence of properties, gppg would need to encode
// such field accesses as ...
// "tmp = get_CurrentSemanticValue(); // Fetch value
// tmp.myField = blah; // update
// set_CurrentSemanticValue(tmp); " // Write update back.
// There is no issue if TValue is restricted to be a ref type.
// The same explanation applies to scanner.yylval.
// ==============================================================
/// <summary>
/// The current value of the "$$" symbolic variable in the parser
/// </summary>
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
protected TValue CurrentSemanticValue;
/// <summary>
/// The current value of the "@$" symbolic variable in the parser
/// </summary>
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
protected TSpan CurrentLocationSpan;
protected int NextToken;
private TSpan LastSpan;
private State FsaState;
private bool recovering;
private int tokensSinceLastError;
private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
/// <summary>
/// The stack of semantic value (YYSTYPE) values.
/// </summary>
protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
/// <summary>
/// The stack of location value (YYLTYPE) varlues.
/// </summary>
protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
private int errorToken;
private int endOfFileToken;
private string[] nonTerminals;
private State[] states;
private Rule[] rules;
/// <summary>
/// Initialization method to allow derived classes
/// to insert the rule list into this base class.
/// </summary>
/// <param name="rules">The array of Rule objects</param>
protected void InitRules( Rule[] rules ) { this.rules = rules; }
/// <summary>
/// Initialization method to allow derived classes to
/// insert the states table into this base class.
/// </summary>
/// <param name="states">The pre-initialized states table</param>
protected void InitStates( State[] states ) { this.states = states; }
/// <summary>
/// OBSOLETE FOR VERSION 1.4.0
/// </summary>
/// <param name="size"></param>
protected void InitStateTable( int size ) { states = new State[size]; }
/// <summary>
/// Initialization method to allow derived classes
/// to insert the special value for the error and EOF tokens.
/// </summary>
/// <param name="err">The error state ordinal</param>
/// <param name="end">The EOF stat ordinal</param>
protected void InitSpecialTokens( int err, int end ) {
errorToken = err;
endOfFileToken = end;
}
/// <summary>
/// Initialization method to allow derived classes to
/// insert the non-terminal symbol names into this base class.
/// </summary>
/// <param name="names">Non-terminal symbol names</param>
protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
#region YYAbort, YYAccept etcetera.
[Serializable]
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
// Reason for FxCop message suppression -
// This exception cannot escape from the local context
private class AcceptException : Exception {
internal AcceptException() { }
protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
}
[Serializable]
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
// Reason for FxCop message suppression -
// This exception cannot escape from the local context
private class AbortException : Exception {
internal AbortException() { }
protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
}
[Serializable]
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
// Reason for FxCop message suppression -
// This exception cannot escape from the local context
private class ErrorException : Exception {
internal ErrorException() { }
protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
}
// The following methods are only called from within
// a semantic action. The thrown exceptions can never
// propagate outside the ShiftReduceParser class in
// which they are nested.
/// <summary>
/// Force parser to terminate, returning "true"
/// </summary>
protected static void YYAccept() { throw new AcceptException(); }
/// <summary>
/// Force parser to terminate, returning "false"
/// </summary>
protected static void YYAbort() { throw new AbortException(); }
/// <summary>
/// Force parser to terminate, returning
/// "false" if error recovery fails.
/// </summary>
protected static void YYError() { throw new ErrorException(); }
/// <summary>
/// Check if parser in error recovery state.
/// </summary>
protected bool YYRecovering { get { return recovering; } }
#endregion
/// <summary>
/// Abstract base method. ShiftReduceParser calls this
/// to initialize the base class data structures. Concrete
/// parser classes must override this method.
/// </summary>
protected abstract void Initialize();
/// <summary>
/// Main entry point of the Shift-Reduce Parser.
/// </summary>
/// <returns>True if parse succeeds, else false for
/// unrecoverable errors</returns>
public bool Parse() {
Initialize(); // allow derived classes to instantiate rules, states and nonTerminals
NextToken = 0;
FsaState = states[0];
StateStack.Push( FsaState );
valueStack.Push( CurrentSemanticValue );
LocationStack.Push( CurrentLocationSpan );
while (true) {
#if TRACE_ACTIONS
Console.Error.WriteLine("Entering state {0} ", FsaState.number);
DisplayStack();
#endif
int action = FsaState.defaultAction;
if (FsaState.ParserTable != null) {
if (NextToken == 0) {
// We save the last token span, so that the location span
// of production right hand sides that begin or end with a
// nullable production will be correct.
LastSpan = scanner.yylloc;
NextToken = scanner.yylex();
#if TRACE_ACTIONS
Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
#endif
}
#if TRACE_ACTIONS
else
Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
#endif
if (FsaState.ParserTable.ContainsKey( NextToken ))
action = FsaState.ParserTable[NextToken];
}
if (action > 0) // shift
{
Shift( action );
}
else if (action < 0) // reduce
{
try {
Reduce( -action );
if (action == -1) // accept
return true;
}
catch (AbortException) {
return false;
}
catch (AcceptException) {
return true;
}
catch (ErrorException) {
if (!ErrorRecovery())
return false;
else
throw; // Rethrow x, preserving information.
}
}
else if (action == 0 && !ErrorRecovery()) // error
return false;
}
}
private void Shift( int stateIndex ) {
#if TRACE_ACTIONS
Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
#endif
FsaState = states[stateIndex];
valueStack.Push( scanner.yylval );
StateStack.Push( FsaState );
LocationStack.Push( scanner.yylloc );
if (recovering) {
if (NextToken != errorToken)
tokensSinceLastError++;
if (tokensSinceLastError > 5)
recovering = false;
}
if (NextToken != endOfFileToken)
NextToken = 0;
}
private void Reduce( int ruleNumber ) {
#if TRACE_ACTIONS
DisplayRule(ruleNumber);
#endif
Rule rule = rules[ruleNumber];
int rhLen = rule.RightHandSide.Length;
//
// Default actions for unit productions.
//
if (rhLen == 1) {
CurrentSemanticValue = valueStack.TopElement(); // Default action: $$ = $1;
CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
}
else if (rhLen == 0) {
// Create a new blank value.
// Explicit semantic action may mutate this value
CurrentSemanticValue = default( TValue );
// The location span for an empty production will start with the
// beginning of the next lexeme, and end with the finish of the
// previous lexeme. This gives the correct behaviour when this
// nonsense value is used in later Merge operations.
CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
scanner.yylloc.Merge( LastSpan ) :
default( TSpan ));
}
else {
// Default action: $$ = $1;
CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
// Default action "@$ = @1.Merge(@N)" for location info.
TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
TSpan atN = LocationStack[LocationStack.Depth - 1];
CurrentLocationSpan =
((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
}
DoAction( ruleNumber );
for (int i = 0; i < rule.RightHandSide.Length; i++) {
StateStack.Pop();
valueStack.Pop();
LocationStack.Pop();
}
FsaState = StateStack.TopElement();
if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
FsaState = states[FsaState.Goto[rule.LeftHandSide]];
StateStack.Push( FsaState );
valueStack.Push( CurrentSemanticValue );
LocationStack.Push( CurrentLocationSpan );
}
/// <summary>
/// Execute the selected action from array.
/// Must be overriden in derived classes.
/// </summary>
/// <param name="actionNumber">Index of the action to perform</param>
protected abstract void DoAction( int actionNumber );
private bool ErrorRecovery() {
bool discard;
if (!recovering) // if not recovering from previous error
ReportError();
if (!FindErrorRecoveryState())
return false;
//
// The interim fix for the "looping in error recovery"
// artifact involved moving the setting of the recovering
// bool until after invalid tokens have been discarded.
//
ShiftErrorToken();
discard = DiscardInvalidTokens();
recovering = true;
tokensSinceLastError = 0;
return discard;
}
private void ReportError() {
StringBuilder errorMsg = new StringBuilder();
errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
if (FsaState.ParserTable.Count < 7) {
bool first = true;
foreach (int terminal in FsaState.ParserTable.Keys) {
if (first)
errorMsg.Append( ", expecting " );
else
errorMsg.Append( ", or " );
errorMsg.Append( TerminalToString( terminal ) );
first = false;
}
}
scanner.yyerror( errorMsg.ToString() );
}
private void ShiftErrorToken() {
int old_next = NextToken;
NextToken = errorToken;
Shift( FsaState.ParserTable[NextToken] );
#if TRACE_ACTIONS
Console.Error.WriteLine("Entering state {0} ", FsaState.number);
#endif
NextToken = old_next;
}
private bool FindErrorRecoveryState() {
while (true) // pop states until one found that accepts error token
{
if (FsaState.ParserTable != null &&
FsaState.ParserTable.ContainsKey( errorToken ) &&
FsaState.ParserTable[errorToken] > 0) // shift
return true;
#if TRACE_ACTIONS
Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
#endif
StateStack.Pop();
valueStack.Pop();
LocationStack.Pop();
#if TRACE_ACTIONS
DisplayStack();
#endif
if (StateStack.IsEmpty()) {
#if TRACE_ACTIONS
Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
#endif
return false;
}
else
FsaState = StateStack.TopElement();
}
}
private bool DiscardInvalidTokens() {
int action = FsaState.defaultAction;
if (FsaState.ParserTable != null) {
// Discard tokens until find one that works ...
while (true) {
if (NextToken == 0) {
#if TRACE_ACTIONS
Console.Error.Write("Reading a token: ");
#endif
NextToken = scanner.yylex();
}
#if TRACE_ACTIONS
Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
#endif
if (NextToken == endOfFileToken)
return false;
if (FsaState.ParserTable.ContainsKey( NextToken ))
action = FsaState.ParserTable[NextToken];
if (action != 0)
return true;
else {
#if TRACE_ACTIONS
Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
#endif
NextToken = 0;
}
}
}
else if (recovering && tokensSinceLastError == 0) {
//
// Boolean recovering is not set until after the first
// error token has been shifted. Thus if we get back
// here with recovering set and no tokens read we are
// looping on the same error recovery action. This
// happens if current_state.ParserTable is null because
// the state has an LR(0) reduction, but not all
// lookahead tokens are valid. This only occurs for
// error productions that *end* on "error".
//
// This action discards tokens one at a time until
// the looping stops. Another attack would be to always
// use the LALR(1) table if a production ends on "error"
//
#if TRACE_ACTIONS
Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
#endif
if (NextToken == endOfFileToken)
return false;
NextToken = 0;
return true;
}
else
return true;
}
/// <summary>
/// Traditional YACC method. Discards the next input token.
/// </summary>
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
protected void yyclearin() { NextToken = 0; }
/// <summary>
/// Tradional YACC method. Clear the "recovering" flag.
/// </summary>
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
protected void yyerrok() { recovering = false; }
/// <summary>
/// OBSOLETE FOR VERSION 1.4.0
/// Method used by derived types to insert new
/// state instances in the "states" array.
/// </summary>
/// <param name="stateNumber">index of the state</param>
/// <param name="state">data for the state</param>
protected void AddState( int stateNumber, State state ) {
states[stateNumber] = state;
state.number = stateNumber;
}
private void DisplayStack() {
Console.Error.Write( "State stack is now:" );
for (int i = 0; i < StateStack.Depth; i++)
Console.Error.Write( " {0}", StateStack[i].number );
Console.Error.WriteLine();
}
private void DisplayRule( int ruleNumber ) {
Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
DisplayProduction( rules[ruleNumber] );
}
private void DisplayProduction( Rule rule ) {
if (rule.RightHandSide.Length == 0)
Console.Error.Write( "/* empty */ " );
else
foreach (int symbol in rule.RightHandSide)
Console.Error.Write( "{0} ", SymbolToString( symbol ) );
Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
}
/// <summary>
/// Abstract state class naming terminal symbols.
/// This is overridden by derived classes with the
/// name (or alias) to be used in error messages.
/// </summary>
/// <param name="terminal">The terminal ordinal</param>
/// <returns></returns>
protected abstract string TerminalToString( int terminal );
private string SymbolToString( int symbol ) {
if (symbol < 0)
return nonTerminals[-symbol - 1];
else
return TerminalToString( symbol );
}
/// <summary>
/// Return text representation of argument character
/// </summary>
/// <param name="input">The character to convert</param>
/// <returns>String representation of the character</returns>
protected static string CharToString( char input ) {
switch (input) {
case '\a': return @"'\a'";
case '\b': return @"'\b'";
case '\f': return @"'\f'";
case '\n': return @"'\n'";
case '\r': return @"'\r'";
case '\t': return @"'\t'";
case '\v': return @"'\v'";
case '\0': return @"'\0'";
default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
}
}
}
/// <summary>
/// Classes implementing this interface must supply a
/// method that merges two location objects to return
/// a new object of the same type.
/// GPPG-generated parsers have the default location
/// action equivalent to "@$ = @1.Merge(@N);" where N
/// is the right-hand-side length of the production.
/// </summary>
/// <typeparam name="TSpan">The Location type</typeparam>
#if EXPORT_GPPG
public interface IMerge<TSpan>
#else
internal interface IMerge<TSpan>
#endif
{
/// <summary>
/// Interface method that creates a location object from
/// the current and last object. Typically used to create
/// a location object extending from the start of the @1
/// object to the end of the @N object.
/// </summary>
/// <param name="last">The lexically last object to merge</param>
/// <returns>The merged location object</returns>
TSpan Merge( TSpan last );
}
/// <summary>
/// This is the default class that carries location
/// information from the scanner to the parser.
/// If you don't declare "%YYLTYPE Foo" the parser
/// will expect to deal with this type.
/// </summary>
#if EXPORT_GPPG
public class LexLocation : IMerge<LexLocation>
#else
[SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
internal class LexLocation : IMerge<LexLocation>
#endif
{
private int startLine; // start line
private int startColumn; // start column
private int endLine; // end line
private int endColumn; // end column
/// <summary>
/// The line at which the text span starts.
/// </summary>
public int StartLine { get { return startLine; } }
/// <summary>
/// The column at which the text span starts.
/// </summary>
public int StartColumn { get { return startColumn; } }
/// <summary>
/// The line on which the text span ends.
/// </summary>
public int EndLine { get { return endLine; } }
/// <summary>
/// The column of the first character
/// beyond the end of the text span.
/// </summary>
public int EndColumn { get { return endColumn; } }
/// <summary>
/// Default no-arg constructor.
/// </summary>
public LexLocation() { }
/// <summary>
/// Constructor for text-span with given start and end.
/// </summary>
/// <param name="sl">start line</param>
/// <param name="sc">start column</param>
/// <param name="el">end line </param>
/// <param name="ec">end column</param>
public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
/// <summary>
/// Create a text location which spans from the
/// start of "this" to the end of the argument "last"
/// </summary>
/// <param name="last">The last location in the result span</param>
/// <returns>The merged span</returns>
public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
}
/// <summary>
/// Abstract scanner class that GPPG expects its scanners to
/// extend.
/// </summary>
/// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
/// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
#if EXPORT_GPPG
public abstract class AbstractScanner<TValue, TSpan>
#else
internal abstract class AbstractScanner<TValue, TSpan>
#endif
where TSpan : IMerge<TSpan> {
/// <summary>
/// Lexical value optionally set by the scanner. The value
/// is of the %YYSTYPE type declared in the parser spec.
/// </summary>
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
// A field must be declared for this value of parametric type,
// since it may be instantiated by a value struct. If it were
// implemented as a property, machine generated code in derived
// types would not be able to select on the returned value.
#pragma warning disable 649
public TValue yylval; // Lexical value: set by scanner
#pragma warning restore 649
/// <summary>
/// Current scanner location property. The value is of the
/// type declared by %YYLTYPE in the parser specification.
/// </summary>
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
public virtual TSpan yylloc {
get { return default( TSpan ); } // Empty implementation allowing
set { /* skip */ } // yylloc to be ignored entirely.
}
/// <summary>
/// Main call point for LEX-like scanners. Returns an int
/// corresponding to the token recognized by the scanner.
/// </summary>
/// <returns>An int corresponding to the token</returns>
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
public abstract int yylex();
/// <summary>
/// Traditional error reporting provided by LEX-like scanners
/// to their YACC-like clients.
/// </summary>
/// <param name="format">Message format string</param>
/// <param name="args">Optional array of args</param>
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
public virtual void yyerror( string format, params object[] args ) { }
}
/// <summary>
/// Encapsulated state for the parser.
/// Opaque to users, visible to the tool-generated code.
/// </summary>
#if EXPORT_GPPG
public class State
{
/// <summary>
/// The number of states in the automaton.
/// </summary>
public int number;
#else
internal class State {
/// <summary>
/// The index of this state in the states array.
/// </summary>
internal int number;
#endif
internal Dictionary<int, int> ParserTable; // Terminal -> ParseAction
internal Dictionary<int, int> Goto; // NonTerminal -> State;
internal int defaultAction; // = 0; // ParseAction
/// <summary>
/// State transition data for this state. Pairs of elements of the
/// goto array associate symbol ordinals with next state indices.
/// The actions array is passed to another constructor.
/// </summary>
/// <param name="actions">The action list</param>c
/// <param name="goToList">Next state data</param>
public State( int[] actions, int[] goToList )
: this( actions ) {
Goto = new Dictionary<int, int>();
for (int i = 0; i < goToList.Length; i += 2)
Goto.Add( goToList[i], goToList[i + 1] );
}
/// <summary>
/// Action data for this state. Pairs of elements of the
/// action array associate action ordinals with each of
/// those symbols that have actions in the current state.
/// </summary>
/// <param name="actions">The action array</param>
public State( int[] actions ) {
ParserTable = new Dictionary<int, int>();
for (int i = 0; i < actions.Length; i += 2)
ParserTable.Add( actions[i], actions[i + 1] );
}
/// <summary>
/// Set the default action for this state.
/// </summary>
/// <param name="defaultAction">Ordinal of the default action</param>
public State( int defaultAction ) {
this.defaultAction = defaultAction;
}
/// <summary>
/// Set the default action and the state transition table.
/// </summary>
/// <param name="defaultAction">The default action</param>
/// <param name="goToList">Transitions from this state</param>
public State( int defaultAction, int[] goToList )
: this( defaultAction ) {
Goto = new Dictionary<int, int>();
for (int i = 0; i < goToList.Length; i += 2)
Goto.Add( goToList[i], goToList[i + 1] );
}
}
/// <summary>
/// Rule representation at runtime.
/// </summary>
#if EXPORT_GPPG
public class Rule
#else
internal class Rule
#endif
{
internal int LeftHandSide; // symbol
internal int[] RightHandSide; // symbols
/// <summary>
/// Rule constructor. This holds the ordinal of
/// the left hand side symbol, and the list of
/// right hand side symbols, in lexical order.
/// </summary>
/// <param name="left">The LHS non-terminal</param>
/// <param name="right">The RHS symbols, in lexical order</param>
public Rule( int left, int[] right ) {
this.LeftHandSide = left;
this.RightHandSide = right;
}
}
/// <summary>
/// Stack utility for the shift-reduce parser.
/// GPPG parsers have three instances:
/// (1) The parser state stack, T = QUT.Gppg.State,
/// (2) The semantic value stack, T = TValue,
/// (3) The location stack, T = TSpan.
/// </summary>
/// <typeparam name="T"></typeparam>
#if EXPORT_GPPG
public class PushdownPrefixState<T>
#else
internal class PushdownPrefixState<T>
#endif
{
// Note that we cannot use the BCL Stack<T> class
// here as derived types need to index into stacks.
//
private T[] array = new T[8];
private int tos = 0;
/// <summary>
/// Indexer for values of the stack below the top.
/// </summary>
/// <param name="index">index of the element, starting from the bottom</param>
/// <returns>the selected element</returns>
public T this[int index] { get { return array[index]; } }
/// <summary>
/// The current depth of the stack.
/// </summary>
public int Depth { get { return tos; } }
internal void Push( T value ) {
if (tos >= array.Length) {
T[] newarray = new T[array.Length * 2];
System.Array.Copy( array, newarray, tos );
array = newarray;
}
array[tos++] = value;
}
internal T Pop() {
T rslt = array[--tos];
array[tos] = default( T );
return rslt;
}
internal T TopElement() { return array[tos - 1]; }
internal bool IsEmpty() { return tos == 0; }
}
}
@@ -1,4 +0,0 @@
global using System;
global using System.Linq;
global using System.Collections.Generic;
global using System.Collections.Immutable;
@@ -8,7 +8,7 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="Google.Protobuf" Version="3.24.3" /> <PackageReference Include="Google.Protobuf" Version="3.24.3" />
<PackageReference Include="YaccLexTools" Version="1.2.2" /> <PackageReference Include="YaccLexTools" Version="1.2.0" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
File diff suppressed because it is too large. Load diff
@@ -9,17 +9,9 @@
<ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" /> <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
<ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" /> <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />
<ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" /> <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
<ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
<ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" /> <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
<ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" /> <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" /> <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
</ItemGroup> </ItemGroup>
<ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
<PackageReference Include="MongoDB.Bson" Version="2.23.1" />
<PackageReference Include="MongoDB.Driver" Version="2.23.1" />
</ItemGroup>
</Project> </Project>
@@ -1,62 +0,0 @@
// using System;
// using System.Diagnostics;
// using BCG;
// using BCG.Utilities.General;
// namespace bts {
// public class PandasStream : ViewStream, IWorker {
// //C:\Users\Owner\AppData\Local\Microsoft\WindowsApps\python.exe
// public PandasStream(ViewStream vs, ViewStreamDatabase vsd, string pandas_code) {
// }
// public PandasStream(ViewStream vs, string pandas_code) :
// this(vs, ViewStreamDatabase.instance, pandas_code) {
// }
// protected override void myOnFirstSubscriber() {
// // string python_code = @"
// // import winsound
// // frequency = 2500 # Set frequency (in Hz)
// // duration = 1000 # Set duration (in milliseconds)
// // winsound.Beep(frequency, duration)
// // ";
// string python_code = @"
// file_path = \"C:\\Temp\\test2.txt\\" # r prefix for raw string to avoid escape characters
// with open(file_path, \"w\") as f:
// f.write(\"Hello, world!\n\")
// ";
// execute_python_code(python_code);
// }
// static void execute_python_code(string python_code) {
// ProcessStartInfo start = new ProcessStartInfo();
// start.FileName = "python"; // Python executable
// start.Arguments = "-c \"" + python_code.Replace("\"", "\\\"") + "\""; // Pass code as argument
// start.UseShellExecute = false;
// start.RedirectStandardOutput = true;
// using (Process process = Process.Start(start)) {
// using (System.IO.StreamReader reader = process.StandardOutput) {
// string result = reader.ReadToEnd();
// Console.Write(result);
// }
// }
// }
// protected override void myOnLastSubscriber() {
// }
// public void doWork() {
// }
// }
// }
@@ -1,79 +0,0 @@
using BCG.Utilities.General;
using System;
namespace BCG {
public class TCPConnectionTestStream : ViewStream, IWorker, ITCPConnectionHandler, ITCPMessageHandler {
class State {
public int n_sends = 0;
public int n_recvs = 0;
}
private readonly string _ip;
private readonly int _port;
private readonly ViewStreamDatabase _vsd;
State _state = new State();
private readonly int _msg_size;
private IBlueTableSchema _schema;
private TimeSpan _send_period;
private ITCPConnection _connection = null;
private byte[] _buff;
public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds) :
this(ip, port, msg_size, send_period_seconds, ViewStreamDatabase.instance) { }
public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds, ViewStreamDatabase vsd) {
_ip = ip;
_port = port;
_msg_size = msg_size;
_vsd = vsd;
_schema = BlueDataUtility.form_schema("TCPClient",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("n_sends", TypeCode.UInt32),
Tuple.Create("n_recvs", TypeCode.UInt32));
_send_period = TimeSpan.FromSeconds(send_period_seconds);
_buff = new byte[msg_size];
for (int i = 0; i < _buff.Length; i++) {
_buff[i] = (byte)i;
}
}
protected override void myOnFirstSubscriber() {
MasterWorker.getInstance(false).addWorker(this, _send_period);
var server_address = new BlutoSubAddress(_ip, _port);
bts.AsyncTCPConnector_APM.connect_to(server_address, this);
}
protected override void myOnLastSubscriber() {
MasterWorker.getInstance(false).removeWorker(this);
}
public void doWork() {
if (_connection != null) {
_connection.send(_buff);
_state.n_sends++;
}
IBlueRow r = new BlueRow(_schema, 0, _state.n_sends, _state.n_recvs);
BlueTable rows = new BlueTable(r.schema);
rows.addRow(r);
var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
myFireEvent(dp);
}
public void onConnection(ITCPConnection connection) {
_connection = connection;
_connection.register(this);
}
public bool isDisposed { get; }
public void onData(RemoteMessage msg) {
bts.Log.Gen.info($"TCPConnectionTestStream received msg: {msg}");
_state.n_recvs++;
}
public void Dispose() {
}
}
}
@@ -1,3 +0,0 @@
file_path = r"C:\Temp\test2.txt" # r prefix for raw string to avoid escape characters
with open(file_path, "w") as f:
f.write("Hello, world!\n")
@@ -1,6 +1,13 @@
namespace BCG; using System;
using System.Data;
public class AddColumnsStream : MapStream { using System.Collections;
public AddColumnsStream(ViewStream vs, string columnDef) using BCG.Utilities;
: base(vs, new ColumnAdder(columnDef)) { } using BCG.Utilities.General;
} namespace BCG
{
public class AddColumnsStream : MapStream
{
public AddColumnsStream(ViewStream vs, string columnDef) : base (vs, new ColumnAdder (columnDef)) {}
}
}
@@ -1,12 +0,0 @@
namespace bts {
using BCG;
using System;
using System.Collections;
public class AddConstantColumnRow {
AddConstantColumnRow(IBlueRow r, Tuple<string, Type, object>[] constant_cols) {
}
}
}
@@ -1,52 +1,73 @@
using System; // using System;
using System.Collections.Generic; // using System.Collections;
using System.Diagnostics.CodeAnalysis; // using BCG.Utilities;
using System.Linq; // using BCG.Evaluator.AddColumnsExpression;
using Bts.ColumnExpressions; // namespace BCG
// {
namespace BCG; // /// <summary>
// /// Provides methods to add columns to the schema
/// <summary> // /// </summary>
/// Provides methods to add columns to the schema // public class ColumnAdder : IStreamMapper
/// </summary> // {
/// <param name="columnsExpression"> // protected IDictionary myNewColumns;
/// String represting the columns to add to the output table // protected IBlueTableSchema mySchema;
/// </param>
public class ColumnAdder : IStreamMapper { // static syntax myParser =
// Store tuples of (column, index) to preserve order // new syntax();
protected readonly IDictionary<string, (ColumnExpression column, int index)> myNewColumns;
protected IBlueTableSchema? mySchema; // /// <summary>
// /// string represting the columns to be added to the output table.
public ColumnAdder(string columnsExpression) => // /// </summary>
myNewColumns = (ColumnExpressionInterpreter.Evaluate(columnsExpression) ?? // /// <param name="colAddExpression"></param>
throw new Exception("Invalid columnn expression: " + columnsExpression)) // public ColumnAdder (string columnDef)
.Select((c, i) => (c, i)) // {
.ToDictionary(c => c.c.Name, c => (c.c, c.i)); // AddColumnsExpression ace;
[MemberNotNull(nameof(mySchema))] // if (DataStreamLoadFileConfig.instance.useStaticParser)
public virtual IBlueRow map(IBlueRow row) { // {
// Only rebuild schema if necessary (i.e., if it's null) // lock (myParser)
mySchema ??= buildSchema(row.schema, myNewColumns); // {
return new AddColumnsRow(mySchema, row); // ace =
} // (AddColumnsExpression)myParser.Parse(columnDef);
// }
[MemberNotNull(nameof(mySchema))] // }
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) => // else
// Always rebuild schema (even if it's already non-null) // {
mySchema = buildSchema(inputSchema, myNewColumns); // syntax parser = new syntax();
// ace =
[MemberNotNull(nameof(mySchema))] // (AddColumnsExpression)parser.Parse(columnDef);
static BlueTableSchema buildSchema( // }
IBlueTableSchema schema,
IDictionary<string, (ColumnExpression column, int index)> newColumns // if (ace == null)
) { // throw new Exception ("Invalid columnn expression: " + columnDef);
ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + newColumns.Count]; // myNewColumns = ace.columns;
schema.columnInfo.CopyTo(cis, 0); // }
foreach (var (col, index) in newColumns.Values) {
var schemaIdx = schema.columnInfo.Length + index; // public virtual IBlueRow map (IBlueRow row)
cis[schemaIdx] = new ColumnInfo(col.Name, schemaIdx, col.Type, col.Value); // {
} // if (mySchema == null)
return new BlueTableSchema(cis, schema.tableName); // myConstructSchema (row.schema);
} // return new AddColumnsRow (mySchema, row);
} // }
// public IBlueTableSchema getOutputSchema (IBlueTableSchema inputSchema)
// {
// myConstructSchema (inputSchema);
// return mySchema;
// }
// private void myConstructSchema (IBlueTableSchema schema)
// {
// ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + myNewColumns.Count];
// int ciIndex = 0;
// foreach (ColumnInfo ci in schema.columnInfo)
// cis[ciIndex++] = ci;
// foreach (DictionaryEntry de in myNewColumns)
// cis[ciIndex] = new ColumnInfo ((string)de.Key, ciIndex ++,
// ((AddColumnsExpression.ConstantInfo)de.Value).t, ((AddColumnsExpression.ConstantInfo)de.Value).v);
// mySchema = new BlueTableSchema (cis, schema.tableName);
// }
// }
// }
@@ -15,11 +15,6 @@ namespace BCG {
// data: // data:
// epochtime, size, serialized data // epochtime, size, serialized data
[DataStreamUserRespresentation("pers")] [DataStreamUserRespresentation("pers")]
// Saves each udpate of a stream into a file. The token is the the unique logical name of the stream,
// which maps the name of the directory where the streams are saved. The directory (3rd arg) is the
// parent directory where the token folder is stored. Each file goes up to rolloevr_size after which a new
// file is created. The file names are token_epoch_datetime
// the first entry in a file is the complete state
public class PersistorStream : ViewStream { public class PersistorStream : ViewStream {
private ViewStream _source_stream; private ViewStream _source_stream;
private string _token; private string _token;
@@ -1,190 +1,190 @@
// using System; using System;
// using System.Collections.Generic; using System.Collections.Generic;
// using System.Linq; using System.Linq;
// using System.Text; using System.Text;
// using System.IO; using System.IO;
// using System.Runtime.Serialization.Formatters.Binary; using System.Runtime.Serialization.Formatters.Binary;
// using BCG.Utilities.General; using BCG.Utilities.General;
// using bts.collections; using bts.collections;
// using System.Reflection.Metadata; using System.Reflection.Metadata;
// namespace BCG { namespace BCG {
// public class PlaybackStream : ViewStream, IWorker { public class PlaybackStream : ViewStream, IWorker {
// private readonly PlaybackType _type; private readonly PlaybackType _type;
// private BlueTable _current_table; private BlueTable _current_table;
// private IClientDeserializer _deserilializer = new ClientUniversalDeserializer(); private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
// // these are the packets already downloaded from the current file // these are the packets already downloaded from the current file
// private List<PlaybackPacket> _packets = new List<PlaybackPacket>(); private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
// //private IEnumerable<FileStream> _file_streams = new List<FileStream>(); //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
// private IEnumerable<string> _file_names; private IEnumerable<string> _file_names;
// private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null); private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
// private DateTime _playback_t0, _playback_t1; private DateTime _playback_t0, _playback_t1;
// private bool _all_files_read = false; private bool _all_files_read = false;
// SnapshotData _snapshot_data = null; // only initialized when in snapshot mode SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
// public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) { public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
// _type = PlaybackType.Streaming; _type = PlaybackType.Streaming;
// _playback_t0 = PersistorStream.parse_datetime(datetime_start); _playback_t0 = PersistorStream.parse_datetime(datetime_start);
// _playback_t1 = PersistorStream.parse_datetime(datetime_end); _playback_t1 = PersistorStream.parse_datetime(datetime_end);
// string token_directory_name = $"{directory_name}\\{token}"; string token_directory_name = $"{directory_name}\\{token}";
// bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}"); bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
// List<string> files_in_range = new List<string>(); List<string> files_in_range = new List<string>();
// if (Directory.Exists(token_directory_name)) { if (Directory.Exists(token_directory_name)) {
// _file_names = _file_names =
// Directory.GetFiles(token_directory_name).Where(f => { Directory.GetFiles(token_directory_name).Where(f => {
// DateTime file_time = PersistorStream.parse_datetime_from_filename(f); DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
// return file_time > _playback_t0 && file_time < _playback_t1; return file_time > _playback_t0 && file_time < _playback_t1;
// }); });
// } }
// } }
// protected override void myOnFirstSubscriber() { protected override void myOnFirstSubscriber() {
// FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read); FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
// _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs); _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
// PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time); PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
// if (snapshot_packet != null) { if (snapshot_packet != null) {
// foreach (PlaybackPacket p in _snapshot_data.packets) { foreach (PlaybackPacket p in _snapshot_data.packets) {
// byte[] ops; byte[] ops;
// Array row_array; Array row_array;
// _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array); _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
// for (int i = 0; i < ops.Length; i++) { for (int i = 0; i < ops.Length; i++) {
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[]; IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
// byte op = ops[i]; byte op = ops[i];
// if (op == StreamOpCode.create) { if (op == StreamOpCode.create) {
// IBlueRowCollection full_table = rows_args[0]; IBlueRowCollection full_table = rows_args[0];
// _current_table = new BlueTable(full_table); _current_table = new BlueTable(full_table);
// } else if (op == StreamOpCode.multipleRemoveAdd) { } else if (op == StreamOpCode.multipleRemoveAdd) {
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[]; //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection; IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection; IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
// BlueDataUtility.absorbChanges(_current_table, dels, adds); BlueDataUtility.absorbChanges(_current_table, dels, adds);
// } else { } else {
// bts.Log.assert(false, $"Oprator: {op} is not expected"); bts.Log.assert(false, $"Oprator: {op} is not expected");
// } }
// if (p.idx == snapshot_packet.idx) { if (p.idx == snapshot_packet.idx) {
// myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema))); myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
// break; break;
// } }
// } }
// } }
// } }
// } }
// private void _load_packets_from_current_file() { private void _load_packets_from_current_file() {
// int curr_idx = _current_fs.Item1; int curr_idx = _current_fs.Item1;
// FileStream curr_fs = _current_fs.Item2; FileStream curr_fs = _current_fs.Item2;
// _packets = new List<PlaybackPacket>(); _packets = new List<PlaybackPacket>();
// _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs)); _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
// } }
// private bool current_file_all_replayed { private bool current_file_all_replayed {
// get { get {
// bool rv = true; bool rv = true;
// _packets.ForEach(p => rv = rv && p.replayed); _packets.ForEach(p => rv = rv && p.replayed);
// return rv; return rv;
// } }
// } }
// protected override void myOnLastSubscriber() { protected override void myOnLastSubscriber() {
// var fs = _current_fs.Item2; var fs = _current_fs.Item2;
// if (fs != null) { if (fs != null) {
// fs.Close(); fs.Close();
// } }
// ViewStreamDatabase.instance.removeWorker(this); ViewStreamDatabase.instance.removeWorker(this);
// } }
// // public event PlaybackStreamProgress onProgress; // public event PlaybackStreamProgress onProgress;
// // public event Action onFileExhausted; // public event Action onFileExhausted;
// private PlaybackPacket _locate_packet_to_replay() { private PlaybackPacket _locate_packet_to_replay() {
// foreach (var p in _packets) { foreach (var p in _packets) {
// // returns the first non replayed packet // returns the first non replayed packet
// if (!p.replayed) if (!p.replayed)
// return p; return p;
// } }
// //else else
// { {
// double secondsElapsed = new BinaryReader(myFileReader).ReadDouble(); double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
// myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed)); myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
// if (onProgress != null) if (onProgress != null)
// onProgress(myFileReader.Position, myFileReader.Length); onProgress(myFileReader.Position, myFileReader.Length);
// } }
// } }
// #region IWorker Members #region IWorker Members
// public void doWork() { public void doWork() {
// if (_all_files_read) if (_all_files_read)
// return; return;
// if (current_file_all_replayed) { if (current_file_all_replayed) {
// _load_next_file(); _load_next_file();
// } }
// PlaybackPacket curr_packet = _locate_packet_to_replay(); PlaybackPacket curr_packet = _locate_packet_to_replay();
// if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway
// byte[] ops; byte[] ops;
// Array row_array; Array row_array;
// // what a stupid interface! // what a stupid interface!
// _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array); _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
// _fire_packet(ops, row_array); _fire_packet(ops, row_array);
// curr_packet.replay_timestamp = PersistorStream.epoch_now(); curr_packet.replay_timestamp = PersistorStream.epoch_now();
// } else { } else {
// PlaybackPacket prev_packet = _get_previous_packet(curr_packet); PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
// // this is the time we need to fire (replay) this packet // this is the time we need to fire (replay) this packet
// long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous; long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
// long now_epoch = PersistorStream.epoch_now(); long now_epoch = PersistorStream.epoch_now();
// // fire when we are past the // fire when we are past the
// if (now_epoch >= time_to_fire) { if (now_epoch >= time_to_fire) {
// byte[] ops; byte[] ops;
// Array row_array; Array row_array;
// _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array); _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
// _fire_packet(ops, row_array); _fire_packet(ops, row_array);
// curr_packet.replay_timestamp = PersistorStream.epoch_now(); curr_packet.replay_timestamp = PersistorStream.epoch_now();
// } }
// } }
// } }
// private void _fire_packet(byte[] ops, Array row_array) { private void _fire_packet(byte[] ops, Array row_array) {
// for (int i = 0; i < ops.Length; i++) { for (int i = 0; i < ops.Length; i++) {
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[]; IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
// byte op = ops[i]; byte op = ops[i];
// if (op == StreamOpCode.create) { if (op == StreamOpCode.create) {
// IBlueRowCollection full_table = rows_args[0]; IBlueRowCollection full_table = rows_args[0];
// _current_table = new BlueTable(full_table); _current_table = new BlueTable(full_table);
// //_current_table.addRows(full_table); //_current_table.addRows(full_table);
// myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) : myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
// (myLastValue as DataPacket).newRows)); (myLastValue as DataPacket).newRows));
// } else if (op == StreamOpCode.multipleRemoveAdd) { } else if (op == StreamOpCode.multipleRemoveAdd) {
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[]; //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection; IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection; IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
// BlueDataUtility.absorbChanges(_current_table, dels, adds); BlueDataUtility.absorbChanges(_current_table, dels, adds);
// myFireEvent(new DataPacket(_current_table, adds, dels)); myFireEvent(new DataPacket(_current_table, adds, dels));
// } else { } else {
// bts.Log.assert(false, $"Oprator: {op} is not expected"); bts.Log.assert(false, $"Oprator: {op} is not expected");
// } }
// } }
// } }
// private void _load_next_file() { private void _load_next_file() {
// var new_idx = _current_fs.Item1 + 1; var new_idx = _current_fs.Item1 + 1;
// if (new_idx >= _file_names.Count()) { if (new_idx >= _file_names.Count()) {
// _all_files_read = true; _all_files_read = true;
// bts.Log.Gen.info("All files read"); bts.Log.Gen.info("All files read");
// return; return;
// } }
// var prev_fs = _current_fs.Item2; var prev_fs = _current_fs.Item2;
// if (prev_fs != null) if (prev_fs != null)
// prev_fs.Close(); prev_fs.Close();
// var new_filename = _file_names.ElementAt(new_idx); var new_filename = _file_names.ElementAt(new_idx);
// bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}"); bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
// var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read); var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
// _current_fs = new Tuple<int, FileStream>(new_idx, new_fs); _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
// _load_packets_from_current_file(); _load_packets_from_current_file();
// } }
// #endregion #endregion
// } }
// } }
@@ -11,27 +11,13 @@ using System.Reflection.Metadata;
namespace BCG { namespace BCG {
public class SnapshotStream : ViewStream { public class SnapshotStream : ViewStream {
public enum SpecificSnapshot {
FirstOfDay,
LastOfDay
};
private BlueTable _current_table; private BlueTable _current_table;
private IClientDeserializer _deserilializer = new ClientUniversalDeserializer(); private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
public DateTime _snapshot_time; public DateTime _snapshot_time;
public string _snapshot_file; public string _snapshot_file;
public IEnumerable<PlaybackPacket> _packets; public IEnumerable<PlaybackPacket> _packets;
public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
DateTime dt;
var parsed = DateTime.TryParse(snapshot_date, out dt);
dt = dt.Date;
bts.Log.assert(parsed);
SpecificSnapshot specific_snapshot_enum;
parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
bts.Log.assert(parsed);
string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
mySetRealSource(rs);
}
public SnapshotStream(string snapshot_dt, string directory_name, string token) { public SnapshotStream(string snapshot_dt, string directory_name, string token) {
_snapshot_time = PersistorStream.parse_datetime(snapshot_dt); _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
@@ -37,20 +37,6 @@ namespace BCG {
public IEnumerable<PlaybackPacket> packets = new List<PlaybackPacket>(); public IEnumerable<PlaybackPacket> packets = new List<PlaybackPacket>();
} }
public static class StreamPersistorUtils { public static class StreamPersistorUtils {
public static string find_snapshot_file(string directory_name, DateTime snapshot_date, SnapshotStream.SpecificSnapshot specific_snapshot) {
DateTime max_dt = DateTime.MinValue;
string rv = String.Empty;
string[] file_names = Directory.GetFiles(directory_name);
for (int i = 0; i < file_names.Count() - 1; i++) {
string fn = file_names[i];
var dt = parse_datetime_from_filename(fn);
if (dt > max_dt) {
max_dt = dt;
rv = fn;
}
}
return rv;
}
public static string find_snapshot_file(string directory_name, DateTime snapshot_time) { public static string find_snapshot_file(string directory_name, DateTime snapshot_time) {
string[] file_names = Directory.GetFiles(directory_name); string[] file_names = Directory.GetFiles(directory_name);
for (int i = 0; i < file_names.Count() - 1; i++) { for (int i = 0; i < file_names.Count() - 1; i++) {
@@ -1,42 +1,47 @@
using System; // using System;
using System.Collections; // using System.Collections;
// using BCG.Utilities;
using BCG.Utilities; // namespace BCG
// {
// /// <summary>
// /// Computes the max for a given column
// /// </summary>
// public class MaxGroupByEvaluator : GroupByEvaluator
// {
namespace BCG { // public MaxGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
/// <summary> // {}
/// Computes the max for a given column
/// </summary>
public class MaxGroupByEvaluator : GroupByEvaluator {
public MaxGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { } // public MaxGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
// : base (colMapEpression, outputCol, groupByCols) {}
public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) // public override IBlueRow evaluate (DataPacket dp)
: base(colMapEpression, outputCol, groupByCols) { } // {
// myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) :
public override IBlueRow evaluate(DataPacket dp) { // myOutputTable;
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : // string hashString = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
myOutputTable; // IBlueRow currMaxRow = myOutputTable.getRow (hashString);
string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols); // IBlueRow maxRow = BlueDataUtility.computeMaxRow (dp.deletedRows, myOutputColumn);
IBlueRow currMaxRow = myOutputTable.getRow(hashString); // if (maxRow != null)
IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn); // {
if (maxRow != null) { // BlueEventLog.logAssert (currMaxRow != null);
bts.Log.assert(currMaxRow != null); // BlueEventLog.logAssert (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) <= 0);
bts.Log.assert(BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) <= 0); // if (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) == 0)
if (BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) == 0) { // {
maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn); // maxRow = BlueDataUtility.computeMaxRow (dp.newRows, myOutputColumn);
if (maxRow == null) // if (maxRow == null)
myOutputTable.removeRow(hashString); // myOutputTable.removeRow (hashString);
else // else
myOutputTable.addRow(maxRow, hashString); // myOutputTable.addRow (maxRow, hashString);
} // }
} // }
maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn); // maxRow = BlueDataUtility.computeMaxRow (dp.addedRows, myOutputColumn);
// if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true, // // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
// otherwise if maxRow > currMaxRow then its replaced // // otherwise if maxRow > currMaxRow then its replaced
if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) > 0)) // if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) > 0))
myOutputTable.addRow(maxRow, hashString); // myOutputTable.addRow (maxRow, hashString);
return myProject(myOutputTable.getRow(hashString)); // return myProject (myOutputTable.getRow (hashString));
} // }
} // }
} // }
@@ -1,15 +1,20 @@
using System; // using System;
namespace BCG { // namespace BCG
public class MaxStream : ViewStream { // {
public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' ')) { // public class MaxStream : ViewStream
} // {
public MaxStream(ViewStream stream, string outputCol, string[] inputCols) { // public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' '))
mySetRealSource(new GroupByStream(stream, // {
new MaxGroupByEvaluator(outputCol, inputCols))); // }
} // public MaxStream (ViewStream stream, string outputCol, string[] inputCols)
// {
// mySetRealSource (new GroupByStream (stream,
// new MaxGroupByEvaluator (outputCol, inputCols)));
// }
public MaxStream(ViewStream stream, string outputCol) // public MaxStream (ViewStream stream, string outputCol)
: this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { } // : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
} // {}
} // }
// }
@@ -1,36 +1,40 @@
using System; // using System;
// using BCG.Utilities;
using BCG.Utilities; // namespace BCG
// {
// public class MinGroupByEvaluator : MaxGroupByEvaluator
// {
// public MinGroupByEvaluator (string outputCol, string[] groupByCols)
// : base (outputCol, groupByCols) {}
namespace BCG { // public MinGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols) :
public class MinGroupByEvaluator : MaxGroupByEvaluator { // base (colMapEpression, outputCol, groupByCols) {}
public MinGroupByEvaluator(string outputCol, string[] groupByCols)
: base(outputCol, groupByCols) { }
public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) : // public override IBlueRow evaluate (DataPacket dp)
base(colMapEpression, outputCol, groupByCols) { } // {
// myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
// string stringRep = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
// IBlueRow currMinRow = myOutputTable.getRow (stringRep);
// IBlueRow minRow = BlueDataUtility.computeMinRow (dp.deletedRows, myOutputColumn);
// if (minRow != null)
// {
// // BlueEventLog.logAssert (currMinRow != null);
// // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
// if (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) == 0)
// {
// minRow = BlueDataUtility.computeMinRow (dp.newRows, myOutputColumn);
// if (minRow == null)
// myOutputTable.removeRow (stringRep);
// else
// myOutputTable.addRow (minRow, stringRep);
// }
// }
// minRow = BlueDataUtility.computeMinRow (dp.addedRows, myOutputColumn);
// if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) < 0))
// myOutputTable.addRow (minRow, stringRep);
// return myOutputTable.getRow (stringRep);
// }
public override IBlueRow evaluate(DataPacket dp) { // }
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable; // }
string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
IBlueRow currMinRow = myOutputTable.getRow(stringRep);
IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
if (minRow != null) {
// BlueEventLog.logAssert (currMinRow != null);
// BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
if (BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) == 0) {
minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
if (minRow == null)
myOutputTable.removeRow(stringRep);
else
myOutputTable.addRow(minRow, stringRep);
}
}
minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) < 0))
myOutputTable.addRow(minRow, stringRep);
return myOutputTable.getRow(stringRep);
}
}
}
@@ -1,13 +1,17 @@
using System; // using System;
namespace BCG { // namespace BCG
public class MinStream : ViewStream { // {
public MinStream(ViewStream stream, string outputCol, string[] inputCol) { // public class MinStream : ViewStream
mySetRealSource(new GroupByStream(stream, // {
new MinGroupByEvaluator(outputCol, inputCol))); // public MinStream (ViewStream stream, string outputCol, string[] inputCol)
} // {
// mySetRealSource (new GroupByStream (stream,
// new MinGroupByEvaluator (outputCol, inputCol)));
// }
public MinStream(ViewStream stream, string outputCol) // public MinStream (ViewStream stream, string outputCol)
: this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { } // : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
} // {}
} // }
// }
@@ -1,82 +1,94 @@
using System; // using System;
namespace BCG { // namespace BCG
/// <summary> // {
/// Computes the sum for a given column // /// <summary>
/// </summary> // /// Computes the sum for a given column
public class SumGroupByEvaluator : GroupByEvaluator { // /// </summary>
public SumGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { } // public class SumGroupByEvaluator : GroupByEvaluator
// {
// public SumGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
// {}
public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) // public SumGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
: base(colMapEpression, outputCol, groupByCols) { } // : base (colMapEpression, outputCol, groupByCols) {}
public override IBlueRow evaluate(DataPacket dp) { // public override IBlueRow evaluate (DataPacket dp)
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable; // {
// myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
return sumRow; // IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
} // return sumRow;
} // }
// }
public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator { // public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator
private string[] myOutputCols, myGroupByCols; // {
public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) { // private string[] myOutputCols, myGroupByCols;
myOutputCols = spaceSeperatedOutputCols.Split(' '); // public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
myGroupByCols = spaceSeperatedGroupByCols.Split(' '); // {
} // myOutputCols = spaceSeperatedOutputCols.Split(' ');
// myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
// }
public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns) { // public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns)
myOutputCols = output_columns; // {
myGroupByCols = groupby_columns; // myOutputCols = output_columns;
} // myGroupByCols = groupby_columns;
// }
#region IGroupByEvaluator Members // #region IGroupByEvaluator Members
public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); } // public IBlueRow evaluate(DataPacket packet) {return BlueDataUtility.computeSum(packet.newRows, myOutputCols);}
public string[] groupByColumns { get { return myGroupByCols; } } // public string[] groupByColumns { get { return myGroupByCols; } }
public void clear() {/*stateless*/} // public void clear() {/*stateless*/}
public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; } // public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) {return schema;}
#endregion // #endregion
} // }
public class SumGroupByEvaluator<T> : GroupByEvaluator { // public class SumGroupByEvaluator<T> : GroupByEvaluator
public SumGroupByEvaluator(string outputCol, string[] groupByCols) // {
: base(outputCol, groupByCols) { } // public SumGroupByEvaluator(string outputCol, string[] groupByCols)
// : base(outputCol, groupByCols)
// { }
public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) // public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
: base(colMapEpression, outputCol, groupByCols) { } // : base(colMapEpression, outputCol, groupByCols) { }
public override IBlueRow evaluate(DataPacket dp) { // public override IBlueRow evaluate(DataPacket dp)
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable; // {
// myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn); // IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
return sumRow; // return sumRow;
} // }
} // }
public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator { // public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator
private string[] myOutputCols, myGroupByCols; // {
public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) { // private string[] myOutputCols, myGroupByCols;
myOutputCols = spaceSeperatedOutputCols.Split(' '); // public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
myGroupByCols = spaceSeperatedGroupByCols.Split(' '); // {
} // myOutputCols = spaceSeperatedOutputCols.Split(' ');
// myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
// }
#region IGroupByEvaluator Members // #region IGroupByEvaluator Members
public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); } // public IBlueRow evaluate(DataPacket packet) {return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);}
public string[] groupByColumns { get { return myGroupByCols; } } // public string[] groupByColumns { get { return myGroupByCols; } }
public void clear() {/*stateless*/} // public void clear() {/*stateless*/}
public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; } // public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) {return schema;}
#endregion // #endregion
} // }
} // }
@@ -1,89 +1,107 @@
using System; // using System;
using System.Collections.Generic; // using System.Collections.Generic;
namespace BCG { // namespace BCG
public class NullMapper : IStreamMapper { // {
private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>(); // public class NullMapper : IStreamMapper
// {
// private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
public NullMapper(string[] valid_cols) { // public NullMapper(string[] valid_cols)
foreach (string c in valid_cols) // {
_valid_cols[c] = true; // foreach (string c in valid_cols)
_valid_cols["id"] = true; // _valid_cols[c] = true;
} // _valid_cols["id"] = true;
// }
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; } // public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
public IBlueRow map(IBlueRow row) { // public IBlueRow map(IBlueRow row)
MutableBlueRow out_row = new MutableBlueRow(row); // {
foreach (string c in row.schema.columnNames) // MutableBlueRow out_row = new MutableBlueRow(row);
if (!_valid_cols.ContainsKey(c)) // foreach (string c in row.schema.columnNames)
out_row.setValue(c, DBNull.Value); // if (!_valid_cols.ContainsKey(c))
return out_row; // out_row.setValue(c, DBNull.Value);
} // return out_row;
} // }
// }
public class SumStream : ViewStream { // public class SumStream : ViewStream
public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) { // {
string[] outputCols = spaceSeparatedOutputCols.Split(' '); // public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
string[] groupByCols = spaceSeparatedGroupByCols.Split(' '); // {
mySetRealSource(new GroupByStream(stream, // string[] outputCols = spaceSeparatedOutputCols.Split(' ');
outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator : // string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols))); // mySetRealSource(new GroupByStream(stream,
} // outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
// new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
// }
public SumStream(ViewStream stream, string[] output_columns) { // public SumStream(ViewStream stream, string[] output_columns)
ViewStream rs = new AddColumnsStream(stream, "fake_col=0"); // {
rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false); // ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
rs = new ProjectStream(rs, "!fake_col"); // rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
mySetRealSource(rs); // rs = new ProjectStream(rs, "!fake_col");
} // mySetRealSource(rs);
// }
public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns) { // public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns)
string[] output_columns = space_seperated_output_columns.Split(' '); // {
string[] groupby_columns = String.IsNullOrEmpty(space_seperated_groupby_columns) ? new string[] { } : space_seperated_groupby_columns.Split(' '); // string[] output_columns = space_seperated_output_columns.Split(' ');
mySetRealSource(new SumStream(stream, output_columns, groupby_columns, force_null_into_other_columns)); // string[] groupby_columns = String.IsNullOrEmpty (space_seperated_groupby_columns) ? new string[]{} : space_seperated_groupby_columns.Split (' ');
} // mySetRealSource (new SumStream (stream, output_columns, groupby_columns, force_null_into_other_columns));
// }
public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns) { // public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns)
ViewStream rs = null; // {
groupby_columns = groupby_columns ?? new string[] { }; // ViewStream rs = null;
if (groupby_columns.Length == 0) { // groupby_columns = groupby_columns ?? new string[] {};
// rs = new AddColumnsStream(stream, "fake_col=0"); // if (groupby_columns.Length == 0)
// rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] { "fake_col" })); // {
// rs = new ProjectStream(rs, "!fake_col"); // rs = new AddColumnsStream(stream, "fake_col=0");
} else { // rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] {"fake_col"}));
rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns)); // rs = new ProjectStream(rs, "!fake_col");
} // }
if (force_null_into_other_columns) { // else
List<string> valid_cols = new List<string>(output_columns); // {
valid_cols.AddRange(groupby_columns); // rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
rs = new MapStream(rs, new NullMapper(valid_cols.ToArray())); // }
} // if (force_null_into_other_columns)
mySetRealSource(rs); // {
} // List<string> valid_cols = new List<string>(output_columns);
// valid_cols.AddRange(groupby_columns);
// rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
// }
// mySetRealSource(rs);
// }
public SumStream(ViewStream stream, string outputCol) // public SumStream (ViewStream stream, string outputCol)
: this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, "fakeCol") { // : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, "fakeCol")
} // {
} // }
// }
public class IntSumStream : ViewStream { // public class IntSumStream : ViewStream
public IntSumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) { // {
string[] outputCols = spaceSeparatedOutputCols.Split(' '); // public IntSumStream (ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
string[] groupByCols = spaceSeparatedGroupByCols.Split(' '); // {
mySetRealSource(new GroupByStream(stream, // string[] outputCols = spaceSeparatedOutputCols.Split(' ');
outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator : // string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols))); // mySetRealSource(new GroupByStream(stream,
} // outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator:
// new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
// }
} // }
public class SumStream<T> : ViewStream { // public class SumStream<T> : ViewStream
public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols) { // {
string[] inputCols = spaceSeparatedInputCols.Split(' '); // public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols)
mySetRealSource(new GroupByStream(stream, // {
new SumGroupByEvaluator<T>(outputCol, inputCols))); // string[] inputCols = spaceSeparatedInputCols.Split(' ');
} // mySetRealSource(new GroupByStream(stream,
} // new SumGroupByEvaluator<T>(outputCol, inputCols)));
} // }
// }
// }
@@ -1,79 +0,0 @@
using BCG;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using ZstdSharp.Unsafe;
namespace bts {
public class RowDiffStream : ViewStream {
private ViewStream _vs;
private readonly string _order_col, _col_1, _col_2;
private readonly int _diff;
private IBlueTableSchema _out_schema;
private readonly string _diff_col_name;
public RowDiffStream(ViewStream vs, string order_col, string col_1, string col_2, int diff, string diff_col_name) {
_vs = vs;
_order_col = order_col;
_col_1 = col_1;
_col_2 = col_2;
_diff = diff;
_diff_col_name = diff_col_name;
}
private string _diff_col_name_perc {
get {
return $"{_diff_col_name}_percentage";
}
}
protected override void myOnFirstSubscriber() {
_vs.onChange += _handle_vs;
}
protected override void myOnLastSubscriber() {
_vs.onChange -= _handle_vs;
}
private void _handle_vs(object dp_obj) {
var dp = dp_obj as DataPacket;
IBlueRowCollection nrows = dp.newRows;
_out_schema = BlueDataUtility.add_column(nrows.schema, _diff_col_name, typeof(double));
_out_schema = BlueDataUtility.add_column(_out_schema, _diff_col_name_perc, typeof(double));
List<IBlueRow> row_list = new List<IBlueRow>();
foreach (IBlueRow r in nrows)
row_list.Add(r);
//OrderBy sorts in ascedning order
var sorted_rows = row_list.OrderBy(r => r[_order_col]).ToList();
var sorted_row_array = sorted_rows.ToArray();
// reverse it to make it descending
sorted_rows.Reverse();
//sort rows so we can do the diff
OrderedBlueTable rv = new OrderedBlueTable(_out_schema);
int i = 0;
foreach (IBlueRow r in sorted_rows) {
MutableBlueRow nr = new MutableBlueRow(_out_schema, r);
//So, in the expression "x - y," where y is subtracted from x, x is the minuend, and y is the subtrahend.
// we assume x - y, and name variables like that
double x = r.get<double>(_col_1);
int y_row_idx = i + _diff;
double diff = System.Double.NaN;
double perc = System.Double.NaN;
if (y_row_idx < sorted_rows.Count()) {
var y = sorted_rows[y_row_idx].get<double>(_col_2);
diff = x - y;
perc = 100.0 * diff / y;
}
nr.setValue(_diff_col_name, diff);
nr.setValue(_diff_col_name_perc, perc);
rv.addRow(nr);
i++;
}
var dp_out = BlueDataUtility.build_dp_inc(new_rows: rv, myLastValue as DataPacket, true);
myFireEvent(dp_out);
}
}
}
@@ -1,31 +0,0 @@
namespace BCG {
using System.Collections.Generic;
public class KeyValueToRelationalStream : ViewStream {
private readonly ViewStream _kv_str;
private readonly string _doc_id_col, _key_col, _val_col;
public KeyValueToRelationalStream(ViewStream kv_str, string doc_id_col, string key_col, string val_col) {
_kv_str = kv_str;
_doc_id_col = doc_id_col;
_key_col = key_col;
_val_col = val_col;
}
protected override void myOnFirstSubscriber() {
_kv_str.onChange += _on_str_update;
}
protected override void myOnLastSubscriber() {
_kv_str.onChange -= _on_str_update;
}
private void _on_str_update(object dp_obj) {
DataPacket dp = dp_obj as DataPacket;
var rows = dp.newRows;
var rv = BlueDataUtility.keyvalue_to_relational<string>(rows, _doc_id_col, _key_col, _val_col);
var dp_out = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket, true);
myFireEvent(dp_out);
}
}
}
@@ -3,37 +3,45 @@ using Google.Protobuf.WellKnownTypes;
using System; using System;
using BCG.Utilities.General; using BCG.Utilities.General;
namespace BCG { namespace BCG
{
// Keeps adding changing rows to the stream, hence preserving the history // Keeps adding changing rows to the stream, hence preserving the history
// and creating a time series, for example this can be used to make bbo updates // and creating a time series, for example this can be used to make bbo updates
// into a time series of prices/qty // into a time series of prices/qty
public class TimeSeriesStream : ViewStream { public class TimeSeriesStream : ViewStream
{
private BlueTable _table; private BlueTable _table;
private ViewStream _stream; private ViewStream _stream;
private int _id = 0; private int _id = 0;
public TimeSeriesStream(ViewStream stream) { public TimeSeriesStream(ViewStream stream)
{
_stream = stream; _stream = stream;
} }
protected override void myOnFirstSubscriber() { protected override void myOnFirstSubscriber()
{
_stream.onChange += new ChangeDelegate(_onstream_change); _stream.onChange += new ChangeDelegate(_onstream_change);
} }
protected override void myOnLastSubscriber() { protected override void myOnLastSubscriber()
{
_stream.onChange -= new ChangeDelegate(_onstream_change); _stream.onChange -= new ChangeDelegate(_onstream_change);
} }
private void _onstream_change(object o) { private void _onstream_change(object o)
{
DataPacket dp = o as DataPacket; DataPacket dp = o as DataPacket;
DateTime now = DateTime.Now; DateTime now = DateTime.Now;
if (_table == null) { if (_table == null)
{
IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName); IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp")); schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
_table = new BlueTable(schema); _table = new BlueTable(schema);
} }
BlueTable added_rows = new BlueTable(_table.schema); BlueTable added_rows = new BlueTable(_table.schema);
foreach (IBlueRow r in dp.addedRows) { foreach (IBlueRow r in dp.addedRows)
{
MutableBlueRow ar = new MutableBlueRow(_table.schema, r); MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
ar.setValue("timestamp", now); ar.setValue("timestamp", now);
ar.setValue("id", _id++); ar.setValue("id", _id++);
@@ -45,6 +53,114 @@ namespace BCG {
} }
} }
public class TimeBucketStream : ViewStream, IStreamMapper
{
private ViewStream _src;
private readonly string _time_stamp_col;
private IBlueTableSchema _schema;
private int _bucket_size_seconds;
public TimeBucketStream(ViewStream input, string time_stamp_col, int bucket_size_seconds)
{
_time_stamp_col = time_stamp_col;
_bucket_size_seconds = bucket_size_seconds;
var vs = new MapStream(input, this);
mySetRealSource(vs);
}
#region IStreamMapper Members
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema)
{
IBlueTableSchema rv = input_schema;
//rv = BlueDataUtility.remove_columns(rv, _time_stamp_col);
rv = BlueDataUtility.add_column(rv, "bucket_number", typeof(int));
return rv;
}
#endregion
#region IDynamicStreamMapper Members
public IBlueRow map(IBlueRow row)
{
DateTime ts = row.get<DateTime>(_time_stamp_col);
TimeSpan time_of_day = (ts - ts.Date);
int seconds = Convert.ToInt32(time_of_day.TotalSeconds);
double buckets = seconds / _bucket_size_seconds;
int bucket_number = Convert.ToInt32(Math.Floor(buckets));
MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
rv.setValue("bucket_number", bucket_number);
return rv;
}
#endregion
}
// public class ThrottleStream : ViewStream, IWorker
// {
// private ViewStreamDatabase _vsd;
// private ViewStream _stream;
// private int _throttle_ms;
// private TimeSpan _min_fire_period;
// private DateTime _last_Fire_time = DateTime.MinValue;
// private DataPacket _unfired_dp;
// public ThrottleStream(ViewStreamDatabase vsd, ViewStream base_stream, int throttle_ms)
// {
// _vsd = vsd;
// _stream = base_stream;
// _throttle_ms = throttle_ms;
// _min_fire_period = TimeSpan.FromMilliSeconds(throttle_ms);
// }
// protected override void myOnFirstSubscriber()
// {
// _vsd.addWorker(this);
// _stream.onChange += new ChangeDelegate(_onstream_change);
// }
// protected override void myOnLastSubscriber()
// {
// _vsd.removeWorker(this);
// _stream.onChange -= new ChangeDelegate(_onstream_change);
// }
// private void _onstream_change(object obj)
// {
// DataPacket dp = obj as DataPacket;
// if (DateTime.Now - _last_Fire_time >= _min_fire_period)
// {
// // ok to fire here
// }
// else
// {
// _last_cached_state = dp.newRows;
// }
// _possibly_fire(dp.newRows);
// }
// public void doWork()
// {
// if (_unfired_dp != null)
// {
// }
// }
// private void _possibly_fire()
// {
// if (_unfired_dp != null)
// {
// if (DateTime.Now - _last_fire_time > _min_fire_period)
// {
// DataPacket dp = BlueDataUtility.build_dp_inc(_unfired_dp.newRows, myLastValue);
// _unfired_dp = null;
// _last_Fire_time = DateTime.Now;
// myFireEvent(dp);
// }
// }
// }
// }
} }
@@ -1,121 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Security.Cryptography.X509Certificates;
using BCG;
using BCG.Utilities.General;
using HtmlAgilityPack;
namespace BCG {
class Program {
static void Main() {
string url = "http://quotes.toscrape.com";
List<string> quotes = ScrapeQuotes(url);
Console.WriteLine("Quotes from http://quotes.toscrape.com:");
foreach (var quote in quotes) {
Console.WriteLine(quote);
}
}
static List<string> ScrapeQuotes(string url) {
List<string> quotes = new List<string>();
// Create a new HtmlWeb instance
HtmlWeb web = new HtmlWeb();
try {
// Load the HTML document from the specified URL
HtmlDocument document = web.Load(url);
// Extract quotes from the HTML document
var quoteNodes = document.DocumentNode.SelectNodes("//span[@class='text']");
if (quoteNodes != null) {
foreach (var quoteNode in quoteNodes) {
string quote = quoteNode.InnerText.Trim();
quotes.Add(quote);
}
}
} catch (Exception ex) {
Console.WriteLine($"Error: {ex.Message}");
}
return quotes;
}
}
public class WebScapperStream : ViewStream, IWorker {
private readonly string _url;
private readonly string _xpath;
private readonly ViewStreamDatabase _vsd;
private IDictionary<string, int> _line_id_map = new Dictionary<string, int>();
private readonly IBlueTableSchema _schema;
public WebScapperStream()
:
this("http://quotes.toscrape.com", "//span[@class='text']", ViewStreamDatabase.instance) { }
public WebScapperStream(string url, string xpath)
: this(url, xpath, ViewStreamDatabase.instance) { }
public WebScapperStream(string url, string xpath, ViewStreamDatabase vsd) {
bts.Log.Gen.info($"WebScapperStream cons url: {url} xpath: {xpath}");
_schema = BlueDataUtility.form_schema($"{url}_{xpath}",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("url", TypeCode.UInt32),
Tuple.Create("line", TypeCode.String));
_url = url;
_xpath = xpath;
_vsd = vsd;
}
protected override void myOnFirstSubscriber() {
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
}
protected override void myOnLastSubscriber() {
MasterWorker.getInstance(false).removeWorker(this);
}
public void doWork() {
var lines = scrape_lines(_url, _xpath);
Func<string, int> get_id = (line) => {
int rv = 0;
if (!_line_id_map.TryGetValue(line, out rv))
_line_id_map.Add(line, _line_id_map.Count);
return rv;
};
BlueTable nrows = new BlueTable(_schema);
foreach (var l in lines) {
bts.Log.Gen.info($"line: {l}");
var id = get_id(l);
BlueRow nr = new BlueRow(_schema, get_id(l), _url, l);
nrows.addRow(nr);
}
var dp = BlueDataUtility.build_dp_inc(nrows, myLastValue as DataPacket, true);
myFireEvent(dp);
}
static List<string> scrape_lines(string url, string xpath) {
List<string> quotes = new List<string>();
// Create a new HtmlWeb instance
HtmlWeb web = new HtmlWeb();
try {
// Load the HTML document from the specified URL
HtmlDocument document = web.Load(url);
// Extract quotes from the HTML document
var quote_nodes = document.DocumentNode.SelectNodes(xpath);
if (quote_nodes != null) {
foreach (var qn in quote_nodes) {
string quote = qn.InnerText.Trim();
quotes.Add(quote);
}
}
} catch (Exception ex) {
Console.WriteLine($"Error: {ex.Message}");
}
return quotes;
}
}
}
@@ -1,212 +0,0 @@
using System;
using System.Collections.Generic;
using MongoDB.Driver;
using MongoDB.Bson;
using BCG;
using BCG.Utilities.General;
using ZstdSharp.Unsafe;
using System.Runtime.CompilerServices;
using mbts.spark;
namespace BCG {
[DataStreamUserRespresentation("mongodb_sql")]
public class MongoDBSQLStream : ViewStream, IWorker {
private ViewStreamDatabase _vsd;
private readonly string _connection_string, _db_name, _collection_name;
private MongoClient _client;
private BlueTable _table;
// Access the database
private IMongoDatabase _database;
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
// mongodb://192.168.1.80:27017
public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
: this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
}
public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
$"db_name: {db_name}");
_connection_string = connection_string;
_db_name = db_name;
_collection_name = collection_name;
_vsd = vsd;
_client = new MongoClient(_connection_string);
_database = _client.GetDatabase(_db_name);
}
protected override void myOnFirstSubscriber() {
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
}
protected override void myOnLastSubscriber() {
MasterWorker.getInstance(false).removeWorker(this);
}
public void doWork() {
_read_mogodb();
}
private void _read_mogodb() {
HighPerformanceTimer timer = new HighPerformanceTimer();
timer.start();
// Access the collection
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
// Fetch all documents from the collection
var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
_table = _table == null ? null : new BlueTable(_table.schema);
// Iterate through the documents and print values
foreach (var document in documents.ToEnumerable()) {
bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
if (_table == null) {
_init_table(document);
}
_add_row(document);
}
timer.stop();
bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
timer.start();
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
timer.stop();
bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
return;
myFireEvent(dp);
}
private void _init_table(BsonDocument doc_row) {
bts.Log.assert(_table == null);
List<ColumnInfo> cis = new List<ColumnInfo>();
cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
foreach (var n in doc_row.Names) {
cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
}
IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
_table = new BlueTable(schema);
}
private void _add_row(BsonDocument doc) {
var key_val = doc["_id"];
var key = key_val.ToString();
int id;
if (!_id_map.TryGetValue(key, out id)) {
id = _id_map.Count;
_id_map.Add(key, id);
}
MutableBlueRow nr = new MutableBlueRow(_table.schema);
nr.setValue("id", id);
foreach (var n in doc.Names) {
var val = doc[n].ToString();
nr.setValue(n, val);
}
_table.addRow(nr);
}
}
[DataStreamUserRespresentation("mongodb")]
public class MongoDBStream : ViewStream, IWorker {
private ViewStreamDatabase _vsd;
private readonly string _connection_string, _db_name, _collection_name;
private MongoClient _client;
private BlueTable _table;
// Access the database
private IMongoDatabase _database;
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
// mongodb://192.168.1.80:27017
public MongoDBStream(string connection_string, string db_name, string collection_name)
: this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
}
public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
$"db_name: {db_name}");
_connection_string = connection_string;
_db_name = db_name;
_collection_name = collection_name;
_vsd = vsd;
_client = new MongoClient(_connection_string);
_database = _client.GetDatabase(_db_name);
}
protected override void myOnFirstSubscriber() {
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
}
protected override void myOnLastSubscriber() {
MasterWorker.getInstance(false).removeWorker(this);
}
public void doWork() {
_read_mogodb();
}
private void _read_mogodb() {
HighPerformanceTimer timer = new HighPerformanceTimer();
timer.start();
// Access the collection
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
// Fetch all documents from the collection
var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
_table = _table == null ? null : new BlueTable(_table.schema);
// Iterate through the documents and print values
foreach (var document in documents.ToEnumerable()) {
bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
if (_table == null) {
_init_table(document);
}
_add_doc(document);
}
timer.stop();
bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
timer.start();
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
timer.stop();
bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
return;
myFireEvent(dp);
}
private void _init_table(BsonDocument doc_row) {
bts.Log.assert(_table == null);
var schema = BlueDataUtility.form_schema($"{_database}_{_table}",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("doc_id", TypeCode.String),
Tuple.Create("key", TypeCode.String),
Tuple.Create("value", TypeCode.String));
_table = new BlueTable(schema);
}
private void _add_doc(BsonDocument doc) {
var doc_id_bson = doc["_id"];
var doc_id = doc_id_bson.ToString();
Func<string, string, int> get_id = (string doc_id, string key) => {
string hash_key = $"{doc_id}_{key}";
int id = 0;
if (!_id_map.TryGetValue(hash_key, out id)) {
id = _id_map.Count;
_id_map.Add(hash_key, id);
}
return id;
};
foreach (var n in doc.Names) {
string key = n;
var id = get_id(doc_id, key);
MutableBlueRow nr = new MutableBlueRow(_table.schema);
var val = doc[n].ToString();
nr.setValue("id", id);
nr.setValue("doc_id", doc_id);
nr.setValue("key", key);
nr.setValue("value", val);
_table.addRow(nr);
}
}
}
}
@@ -1,34 +0,0 @@
using BCG;
namespace bts {
public class SchemaGuesserStream : ViewStream {
private IBlueRowCollection _rows = null;
private ViewStream _vs;
public SchemaGuesserStream(ViewStream vs) {
_vs = vs;
}
protected override void myOnFirstSubscriber() {
_vs.onChange += _on_data;
}
protected override void myOnLastSubscriber() {
_vs.onChange -= _on_data;
}
private void _on_data(object obj) {
DataPacket dp = obj as DataPacket;
if (_rows == null) {
_rows = BlueDataUtility.get_typed_table(dp.newRows);
}
var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
myFireEvent(dp_out);
}
}
}
@@ -1,107 +1,115 @@
using Bts.ColumnSortExpressions; // using System;
// using System.Collections;
using System; // using BCG.Utilities;
using System.Collections; // using BCG.Evaluator.SortExpression;
using System.Collections.Generic;
using System.Linq;
// namespace BCG
namespace BCG; // {
// public class SortRow : IComparable, IBlueRow
public class SortRow : IComparable, IBlueRow { // {
readonly IBlueRow row; // private IBlueRow myRow;
readonly IEnumerable<ColumnSortExpression>? columns; // private IEnumerable myColumns;
readonly IComparer? comparer; // private IComparer myComparer;
public SortRow(IBlueRow row, IList<ColumnSortExpression> columns) { // public SortRow (IBlueRow row, SortExpression sortExpression)
this.row = row; // {
this.columns = columns; // myRow = row;
} // myColumns = sortExpression.columns;
// }
public SortRow(IBlueRow row, IComparer comparer) {
this.row = row; // public SortRow (IBlueRow row, IEnumerable columns)
this.comparer = comparer; // {
} // myRow = row;
// myColumns = columns;
public IBlueRow internal_row { get { return row; } } // }
#region IComparable method // public SortRow (IBlueRow row, IComparer comparer)
public int CompareTo(object? obj) { // {
if (obj is not IBlueRow otherRow) { // myRow = row;
throw new ArgumentException( // myComparer = comparer;
$"Expected obj to be of type {typeof(IBlueRow)} (actually {obj?.GetType()})", // }
nameof(obj));
} // public IBlueRow internal_row { get { return myRow; } }
if (comparer is not null) {
return comparer.Compare(this, otherRow); // #region IComparable method
} // public int CompareTo (object obj)
return columns.empty_if_null().Select(Compare).FirstOrDefault(c => c != 0); // {
int Compare(ColumnSortExpression col) { // IBlueRow otherRow = (IBlueRow)obj;
var otherVal = otherRow[col.Name]; // if (myComparer == null)
int rv = myIsNull(this[col.Name]) ? // {
(myIsNull(otherVal) ? 0 : -1) : // foreach (SortExpression.ColumnCriteria cc in myColumns)
(myIsNull(otherVal) ? 1 : ((IComparable)this[col.Name]).CompareTo(otherVal)); // {
rv *= col.IsAscending ? 1 : -1; // int retVal = myIsNull (this[cc.columnName]) ?
return rv; // myIsNull (otherRow[cc.columnName]) ? 0 : -1 :
} // myIsNull (otherRow[cc.columnName]) ? 1 : ((IComparable)this[cc.columnName]).CompareTo (otherRow[cc.columnName]);
} // retVal *= cc.ascending ? 1 : -1;
#endregion // if (retVal != 0)
// return retVal;
#region IBlueRow members // }
public IBlueTableSchema schema { get { return row.schema; } } // return 0;
#endregion // }
// else
#region ICollection members // return myComparer.Compare (this, otherRow);
public int Count { get { return row.Count; } } // }
// #endregion
public bool IsSynchronized { get { return false; } }
// #region IBlueRow members
public object SyncRoot { get { return this; } } // public IBlueTableSchema schema{get{return myRow.schema;}}
// #endregion
public void CopyTo(Array array, int index) { row.CopyTo(array, index); }
#endregion // #region ICollection members
// public int Count {get{return myRow.Count;}}
#region IDictionary members
public bool IsFixedSize { get { return true; } } // public bool IsSynchronized {get{return false;}}
public bool IsReadOnly { get { return true; } } // public object SyncRoot {get{return this;}}
/// <summary> // public void CopyTo (Array array, int index) {myRow.CopyTo (array, index);}
/// The key can either be the string corrsponding to the column name or // #endregion
/// the order of the column
/// </summary> // #region IDictionary members
public object this[object key] { // public bool IsFixedSize {get{return true;}}
get {
if (key is string) // public bool IsReadOnly {get{return true;}}
return this[(string)key];
return this[(int)key]; // /// <summary>
} // /// The key can either be the string corrsponding to the column name or
set { throw new NotSupportedException("The IBlueRow is readonly"); } // /// the order of the column
} // /// </summary>
// public object this [object key]
public object this[int index] => row[index]; // {
// get
public object this[string colName] => row[colName]; // {
// if (key is string)
public ICollection Keys => row.Keys; // return this[(string)key];
// return this[(int)key];
public ICollection Values => row.Values; // }
// set {throw new NotSupportedException ("The IBlueRow is readonly");}
public bool Contains(object key) => row.Contains(key); // }
void IDictionary.Add(object key, object? value) => // public object this [int index] {get{return myRow[index];}}
throw new NotSupportedException("The IBlueRow is readonly");
// public object this [string colName] {get{return myRow[colName];}}
void IDictionary.Clear() =>
throw new NotSupportedException("The IBlueRow is readonly"); // public ICollection Keys{get{return myRow.Keys;}}
IDictionaryEnumerator IDictionary.GetEnumerator() { return row.GetEnumerator(); } // public ICollection Values{get{return myRow.Values;}}
IEnumerator IEnumerable.GetEnumerator() { return row.GetEnumerator(); } // public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); } // public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
#endregion
// public bool Contains (object key){return myRow.Contains (key);}
bool myIsNull(object? o) =>
o is null || o == null || o is DBNull; // IDictionaryEnumerator IDictionary.GetEnumerator() {return myRow.GetEnumerator();}
} // IEnumerator IEnumerable.GetEnumerator () {return myRow.GetEnumerator();}
// public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
// #endregion
// private bool myIsNull (object o) {return o == null || o is DBNull;}
// }
// }
@@ -1,99 +1,129 @@
using Bts.ColumnSortExpressions; // using System;
// using System.Collections;
// using BCG.Evaluator.SortExpression;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
namespace BCG; // namespace BCG
// {
// /// <summary>
// /// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
// /// to merge the additions to the accumulates table
// /// </summary>
// public class SortStream : ViewStream
// {
// private ViewStream myStream;
// private IEnumerable mySortCols;
// private IComparer myComparer;
// private DataPacket myLastFiredPacket;
// private DataPacket myLastRecievedPacket;
/// <summary> // public SortStream (ViewStream stream, string sortExpression)
/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort // {
/// to merge the additions to the accumulates table // myStream = stream;
/// </summary> // mySetSortExpression (sortExpression);
public class SortStream : ViewStream { // }
ViewStream stream;
IList<ColumnSortExpression> sortCols;
IComparer? comparer;
DataPacket? lastFiredPacket;
DataPacket? lastReceivedPacket;
public SortStream(ViewStream stream, string sortExpression) { // public SortStream (ViewStream vs, string colName, bool isAsc)
this.stream = stream; // {
SortExpression = sortExpression; // myStream = vs;
} // string sortExpression = isAsc ? "+" + colName : "-" + colName;
// mySetSortExpression (sortExpression);
// }
public SortStream(ViewStream vs, string colName, bool isAsc) { // public SortStream (ViewStream vs, IComparer comparer)
stream = vs; // {
SortExpression = isAsc ? "+" + colName : "-" + colName; // myStream = vs;
} // myComparer = comparer;
// }
public SortStream(ViewStream stream, IComparer comparer) { // public string sortExpression
this.stream = stream; // {
this.comparer = comparer; // set
sortCols = Array.Empty<ColumnSortExpression>(); // {
} // myComparer = null;
// myLastFiredPacket = null;
// mySetSortExpression (value);
// myChangeHandler (myLastRecievedPacket);
// }
// }
[MemberNotNull(nameof(sortCols))] // public IComparer comparer
public string SortExpression { // {
set { // set
comparer = null; // {
lastFiredPacket = null; // myComparer = value;
sortCols = ColumnSortExpressionInterpreter.Evaluate(value); // myLastFiredPacket = null;
if (lastReceivedPacket is not null) { // myChangeHandler(myLastRecievedPacket);
myChangeHandler(lastReceivedPacket); // }
} // }
}
}
public IComparer Comparer { // private void mySetSortExpression (string sortExpression)
set { // {
comparer = value; // syntax parser =
lastFiredPacket = null; // new syntax();
if (lastReceivedPacket is not null) {
myChangeHandler(lastReceivedPacket); // mySortCols = ((SortExpression)
} // parser.Parse (sortExpression)).columns;
} // }
}
public void myChangeHandlerTest(object o) => // protected override void myOnFirstSubscriber()
myChangeHandler(o); // {
// myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler);
// }
[MemberNotNull(nameof(lastFiredPacket))] // protected override void myOnLastSubscriber()
internal void myChangeHandler(object o) { // {
lastReceivedPacket = (DataPacket)o; // myInit ();
// First callback, so we need to sort the whole table, this is the // myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
// only time we will do that for all the other sorting operations would // }
// be sorting the increments and inserting them into the whole list
if (lastFiredPacket is null) {
lastFiredPacket = new() {
newRows = comparer is null ?
new SortedTable(lastReceivedPacket.newRows, sortCols) :
new SortedTable(lastReceivedPacket.newRows, comparer)
};
lastFiredPacket.addedRows = new OrderedBlueTable(lastFiredPacket.newRows);
lastFiredPacket.deletedRows = new BlueTable(lastFiredPacket.newRows.schema);
myFireEvent(lastFiredPacket.Clone());
return;
}
// Deletes
lastFiredPacket.deletedRows =
comparer == null ?
new SortedTable(lastReceivedPacket.deletedRows, sortCols) :
new SortedTable(lastReceivedPacket.deletedRows, comparer);
((SortedTable)lastFiredPacket.newRows).deleteRows(lastFiredPacket.deletedRows);
// Adds
lastFiredPacket.addedRows = new BlueTable(lastReceivedPacket.addedRows);
((SortedTable)lastFiredPacket.newRows).addRows(lastFiredPacket.addedRows);
myFireEvent(lastFiredPacket.Clone());
}
protected override void myOnFirstSubscriber() => // internal void myChangeHandler (object o)
stream.onChange += myChangeHandler; // {
// myLastRecievedPacket = (DataPacket)o;
// //first callback, so we need to sort the whole table, this is the
// //only time we will do that for all the other sorting operations would
// //be sorting the increments and inserting them into the whole list
// if (myLastFiredPacket == null)
// {
// myLastFiredPacket = new DataPacket ();
// myLastFiredPacket.newRows =
// myComparer == null ?
// new SortedTable (myLastRecievedPacket.newRows, mySortCols) :
// new SortedTable (myLastRecievedPacket.newRows, myComparer);
// myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
// myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
// myFireEvent (myLastFiredPacket.Clone());
// return;
// }
// myHandleDeletes (myLastRecievedPacket);
// myHandleAdds (myLastRecievedPacket);
// myFireEvent (myLastFiredPacket.Clone());
// }
protected override void myOnLastSubscriber() { // public void myChangeHandlerTest(object o)
lastFiredPacket = null; // {
lastReceivedPacket = null; // myChangeHandler(o);
stream.onChange -= myChangeHandler; // }
}
} // private void myHandleDeletes (DataPacket dp)
// {
// myLastFiredPacket.deletedRows =
// myComparer == null ?
// new SortedTable (dp.deletedRows, mySortCols) :
// new SortedTable (dp.deletedRows, myComparer);
// ((SortedTable)(myLastFiredPacket.newRows)).deleteRows (myLastFiredPacket.deletedRows);
// }
// private void myHandleAdds (DataPacket dp)
// {
// myLastFiredPacket.addedRows = new BlueTable (dp.addedRows);
// ((SortedTable)(myLastFiredPacket.newRows)).addRows (myLastFiredPacket.addedRows);
// }
// private void myInit ()
// {
// myLastFiredPacket = null;
// myLastRecievedPacket = null;
// }
// }
// }
@@ -1,108 +1,136 @@
using System; // using System;
using System.Collections; // using System.Collections;
using System.Collections.Generic; // using BCG.Evaluator.SortExpression;
using System.Diagnostics.CodeAnalysis;
// namespace BCG
using Bts.ColumnSortExpressions; // {
// /// <summary>
namespace BCG; // /// Represents an IBlueRowCollection with the rows sorted according to the
// ///
/// <summary> // /// </summary>
/// Represents an IBlueRowCollection with the rows sorted according to the // public class SortedTable : IBlueRowCollection
/// </summary> // {
public class SortedTable : IBlueRowCollection { // private ArrayList myRows = new ArrayList ();
readonly List<SortRow> rows = new(0); // private Hashtable myIndex = new Hashtable ();
readonly Dictionary<int, IBlueRow> rowIndex = new(0); // private IBlueTableSchema mySchema;
readonly IList<ColumnSortExpression> sortCols = Array.Empty<ColumnSortExpression>(); // private IEnumerable mySortCols;
readonly IComparer? comparer; // private IComparer myComparer;
/// <summary> // #region SortedTable method(s)
/// Sorts the rows using sortExpression as the sorting criteria // public void addRows (IBlueRowCollection rows)
/// </summary> // {
/// <param name="rows">Rows to be inserted into the table</param> // //requires merger of the addedRows to the already existing ones
/// <param name="sortCols">Sorting criteria</param> // if (rows.Count == 0)
public SortedTable(IBlueRowCollection rows, IList<ColumnSortExpression> sortCols) { // //at times there are calls with 0 additions, in that case
this.sortCols = sortCols; // //we ofcourse need no processing
init(rows); // return;
} // SortedTable addedRows =
// myComparer == null ? new SortedTable (rows, mySortCols) :
public SortedTable(IBlueRowCollection rows, IComparer comparer) { // new SortedTable (rows, myComparer);
this.comparer = comparer; // int addedRowsIndex = 0;
init(rows); // SortRow newRow = addedRows.getRowByIndex (addedRowsIndex);
} // for (int i = 0; i < myRows.Count; i ++)
// if (newRow.CompareTo(myRows[i]) <= 0)
public int Count => rows.Count; // {
public bool IsSynchronized => false; // myRows.Insert (i, newRow);
public IBlueTableSchema schema { get; private set; } // myIndex[newRow["id"]] = newRow;
public object SyncRoot => this; // if (++addedRowsIndex == addedRows.Count)
public IBlueRow this[int id] => rowIndex[id] // return;
?? throw new BlueDataException( // newRow = addedRows.getRowByIndex (addedRowsIndex);
$"ID '{id}' does not exist in table '{schema?.tableName}'"); // }
// if (addedRowsIndex < addedRows.Count)
public void addRows(IBlueRowCollection rows) { // for (int i = addedRowsIndex; i < addedRows.Count; i++)
// Requires merger of the addedRows to the already existing ones // {
if (rows.Count == 0) { // newRow = addedRows.getRowByIndex (i);
// At times there are calls with 0 additions, in that case // myRows.Add (newRow);
// we of course need no processing // myIndex[newRow["id"]] = newRow;
return; // }
} // }
SortedTable addedRows =
comparer == null ? new SortedTable(rows, sortCols) : // public void deleteRows (IBlueRowCollection deletedRows)
new SortedTable(rows, comparer); // {
int addedRowsIndex = 0; // foreach (IBlueRow deletedRow in deletedRows)
SortRow newRow = addedRows.getRowByIndex(addedRowsIndex); // {
for (int i = 0; i < this.rows.Count; i++) { // IBlueRow row = (SortRow)myIndex[deletedRow["id"]];
if (newRow.CompareTo(this.rows[i]) <= 0) { // myRows.Remove (row);
this.rows.Insert(i, newRow); // myIndex.Remove (row["id"]);
rowIndex[(int)newRow["id"]] = newRow; // }
if (++addedRowsIndex == addedRows.Count) { // }
return;
} // public SortRow getRowByIndex (int index) {return (SortRow)myRows[index];}
newRow = addedRows.getRowByIndex(addedRowsIndex); // #endregion
}
} // #region Constructor(s)
if (addedRowsIndex < addedRows.Count) { // /// <summary>
for (int i = addedRowsIndex; i < addedRows.Count; i++) { // /// Sorts the rows using sortExpression as the sorting criteria
newRow = addedRows.getRowByIndex(i); // /// </summary>
this.rows.Add(newRow); // /// <param name="rows">
rowIndex[(int)newRow["id"]] = newRow; // /// Rows to be inserted into the table
} // /// </param>
} // /// <param name="sortExpression">
} // /// Sorting criteria
// /// </param>
public void deleteRows(IBlueRowCollection deletedRows) { // public SortedTable (IBlueRowCollection rows, SortExpression sortExpression)
foreach (IBlueRow deletedRow in deletedRows) { // {
var row = rowIndex[(int)deletedRow["id"]!] as SortRow // mySortCols = sortExpression.columns;
?? throw new InvalidOperationException("Index did not return IBlueRow"); // myConstructorHelper (rows);
rows.Remove(row); // }
rowIndex.Remove((int)row["id"]!);
} // public SortedTable (IBlueRowCollection rows, IEnumerable sortCols)
} // {
// mySortCols = sortCols;
public SortRow getRowByIndex(int index) => // myConstructorHelper (rows);
rows[index]!; // }
public bool containsID(int id) => // public SortedTable (IBlueRowCollection rows, IComparer comparer)
rowIndex[id] is not null; // {
// myComparer = comparer;
public IEnumerator GetEnumerator() => // myConstructorHelper (rows);
rows.GetEnumerator(); // }
public void CopyTo(Array array, int index) { // private void myConstructorHelper (IBlueRowCollection rows)
for (int i = 0; i < Count; i++) { // {
array.SetValue(rows[i], index + i); // mySchema = rows.schema;
} // foreach (IBlueRow row in rows)
} // {
// SortRow newRow =
[MemberNotNull(nameof(schema))] // myComparer == null ?
void init(IBlueRowCollection rows) { // new SortRow (row, mySortCols) : new SortRow (row, myComparer);
schema = rows.schema; // myRows.Add (newRow);
foreach (IBlueRow row in rows) { // myIndex[row["id"]] = newRow;
SortRow newRow = comparer is null ? // }
new(row, sortCols) : new(row, comparer); // myRows.Sort ();
this.rows.Add(newRow); // }
rowIndex[(int)row["id"]!] = newRow;
} // #endregion
this.rows.Sort();
} // #region IBlueRowCollections members
} // public IBlueRow this[int id]
// {
// get
// {
// if (myIndex[id] == null)
// throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
// return (IBlueRow)myIndex[id];
// }
// }
// public IBlueTableSchema schema {get{return mySchema;}}
// public bool containsID (int id) {return myIndex[id] != null;}
// #endregion
// #region ICollection members
// public IEnumerator GetEnumerator () {return myRows.GetEnumerator ();}
// public bool IsSynchronized {get{return false;}}
// public object SyncRoot {get{return this;}}
// public void CopyTo (Array array, int index){myRows.CopyTo (array, index);}
// public int Count{get{return myRows.Count;}}
// #endregion
// }
// }
@@ -1,43 +1,49 @@
using System; // using System;
namespace BCG { // namespace BCG
public class TopStream : MethodDrivenStream { // {
private int myTopCount; // public class TopStream : MethodDrivenStream
// {
// private int myTopCount;
public TopStream(ViewStream stream, int topCount) : base(stream) { myTopCount = topCount; } // public TopStream (ViewStream stream, int topCount) : base (stream) {myTopCount = topCount;}
public TopStream(ViewStream stream, // public TopStream (ViewStream stream,
int topCount, // int topCount,
string sortExpression) // string sortExpression)
: // :
this(sortExpression == null ? // this (sortExpression == null?
stream : // stream :
new SortStream(stream, sortExpression), topCount) { } // new SortStream (stream, sortExpression), topCount)
// {}
protected override void myOnTableClear() { // protected override void myOnTableClear ()
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows; // {
myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema); // myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema); // myLastFiredPacket.newRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
} // myLastFiredPacket.addedRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
// }
protected override void myHandleFirstCall() { // protected override void myHandleFirstCall ()
myLastFiredPacket = new DataPacket(); // {
myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount); // myLastFiredPacket = new DataPacket ();
myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows); // myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema); // myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
} // myLastFiredPacket.deletedRows = new OrderedBlueTable (myLastFiredPacket.newRows.schema);
// }
protected override void myHandleIncrementalAdd() { } // protected override void myHandleIncrementalAdd () {}
protected override void myHandleIncrementalDelete() { // protected override void myHandleIncrementalDelete()
IBlueRowCollection oldRows = myLastFiredPacket.newRows; // {
myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount); // IBlueRowCollection oldRows = myLastFiredPacket.newRows;
BlueDataUtility.computeDifferences(myLastFiredPacket.newRows, // myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
oldRows, // BlueDataUtility.computeDifferences (myLastFiredPacket.newRows,
myLastFiredPacket.schema.columnNames, // oldRows,
out myLastFiredPacket.deletedRows, // myLastFiredPacket.schema.columnNames,
out myLastFiredPacket.addedRows); // out myLastFiredPacket.deletedRows,
myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true; // out myLastFiredPacket.addedRows);
} // myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
} // }
} // }
// }
File diff suppressed because it is too large. Load diff
@@ -11,17 +11,17 @@ namespace BCG
public class CircularRowCollection : OrderedBlueTable public class CircularRowCollection : OrderedBlueTable
{ {
private int myBufferSize; private int myBufferSize;
public CircularRowCollection(IBlueTableSchema schema, int bufferSize) public CircularRowCollection (IBlueTableSchema schema, int bufferSize)
: base(schema) : base (schema)
{ {
myBufferSize = bufferSize; myBufferSize = bufferSize ;
} }
public CircularRowCollection(IBlueTableSchema schema, IEnumerable<IBlueRow> rows, int bufferSize) public CircularRowCollection (IBlueRowCollection rows, int bufferSize)
: this(schema, bufferSize) : this (rows.schema, bufferSize)
{ {
foreach (IBlueRow r in rows) foreach (IBlueRow r in rows)
addRow(r); addRow (r);
} }
public int bufferSize public int bufferSize
@@ -39,30 +39,30 @@ namespace BCG
/// </summary> /// </summary>
/// <param name="r"></param> /// <param name="r"></param>
/// <returns></returns> /// <returns></returns>
public IBlueRow addRowWithReturn(IBlueRow r) public IBlueRow addRowWithReturn (IBlueRow r)
{ {
base.addRow(r); base.addRow (r);
if (myOrderedRows.Count > myBufferSize) if (myOrderedRows.Count > myBufferSize)
{ {
IBlueRow retVal = myOrderedRows[0] as IBlueRow; IBlueRow retVal = myOrderedRows [0] as IBlueRow;
deleteRow(retVal); deleteRow (retVal);
return retVal; return retVal;
} }
return null; return null;
} }
public override void addRow(IBlueRow row) public override void addRow (IBlueRow row)
{ {
base.addRow(row); base.addRow (row);
if (myOrderedRows.Count > myBufferSize) if (myOrderedRows.Count > myBufferSize)
deleteRow(myOrderedRows[0] as IBlueRow); deleteRow (myOrderedRows [0] as IBlueRow);
} }
public override void addRow(DataRow row) public override void addRow (DataRow row)
{ {
base.addRow(row); base.addRow (row);
if (myOrderedRows.Count > myBufferSize) if (myOrderedRows.Count > myBufferSize)
deleteRow(myOrderedRows[0] as IBlueRow); deleteRow (myOrderedRows [0] as IBlueRow);
} }
public void add_rows(IEnumerable<IBlueRow> rows_to_add, out IBlueRowCollection added_rows, out IBlueRowCollection deleted_rows) public void add_rows(IEnumerable<IBlueRow> rows_to_add, out IBlueRowCollection added_rows, out IBlueRowCollection deleted_rows)
@@ -3,7 +3,8 @@ using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Text; using System.Text;
namespace BCG { namespace BCG
{
/* /*
These fields are first going to be used by BlueRay to help it determine what differnet tags mean, but eventually These fields are first going to be used by BlueRay to help it determine what differnet tags mean, but eventually
* the goal is that they will be used for all services in this system. * the goal is that they will be used for all services in this system.
@@ -11,12 +12,11 @@ namespace BCG {
* not all metadata fields are present in all metadata; it depends upon the class. For example, a bluepages service * not all metadata fields are present in all metadata; it depends upon the class. For example, a bluepages service
* probably won't have a 'symbol' or 'exchange' field populated, but it'll probably have the 'type' of service populated * probably won't have a 'symbol' or 'exchange' field populated, but it'll probably have the 'type' of service populated
*/ */
public enum MetaDataField { public enum MetaDataField { Protocol, //is this a bluepages service, a pubsub tag, a blueray tag, a 29 west thing, or what
Protocol, // is this a bluepages service, a pubsub tag, a blueray tag, a 29 west thing, or what Type, //if it's a multicast, is it option, fireprice, whatever. If it's bluepages, is it persistence, pubsub, whatever
Type, // if it's a multicast, is it option, fireprice, whatever. If it's bluepages, is it persistence, pubsub, whatever Symbol,
Symbol, Exchange,
Exchange, Platform,
MarketDataType, // BAM (bid, ask, mid), last, etc MarketDataType, // BAM (bid, ask, mid), last, etc
RealName RealName }; //does this thing have a better name we could call it
}; //does this thing have a better name we could call it
} }
@@ -74,7 +74,7 @@ namespace BCG {
break; break;
} }
if (isVisible) { if (isVisible) {
Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName); Log.assert(myTypeLookupTable[userRep] == null);
myTypeLookupTable[userRep] = type; myTypeLookupTable[userRep] = type;
} }
} }
@@ -8,7 +8,7 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="Google.Protobuf" Version="3.24.3" /> <PackageReference Include="Google.Protobuf" Version="3.24.3" />
<PackageReference Include="YaccLexTools" Version="1.2.2" /> <PackageReference Include="YaccLexTools" Version="1.2.0" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
File diff suppressed because it is too large. Load diff
@@ -6,17 +6,16 @@ using Bts.DataStreamExpressions;
using System.Collections.Generic; using System.Collections.Generic;
using BCG; using BCG;
using System.IO;
using BCG.Utilities.General;
using bts;
using System.Linq;
bts.Log.Gen.info($"Starting DSPublisher"); bts.Log.Gen.info($"Starting DSPublisher");
Console.WriteLine("Starting DSPublisher..."); Console.WriteLine("Starting DSPublisher...");
bool run_echo_tcp_client_only = false; static void init(string[] args)
static void init(string[] args) { {
Func<string, string, Task> login = async (l, p) => { Console.WriteLine("Hit key to continue");
Console.ReadLine();
Func<string, string, Task> login = async (l, p) =>
{
var login = l; var login = l;
var passwd = p; var passwd = p;
var service_ctx = bts.api.core.ServiceContext.create(); var service_ctx = bts.api.core.ServiceContext.create();
@@ -26,105 +25,60 @@ static void init(string[] args) {
Configuration.initialize(); Configuration.initialize();
Task.Run(async () => await login("bts", String.Empty)); Task.Run(async () => await login("bts", String.Empty));
} }
init(args);
if (run_echo_tcp_client_only)
start_echo_client();
else
_publish_streams();
static ViewStream build_stream(string stream_expression) { init(args);
try { _publish_streams();
bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}");
static ViewStream build_stream(string stream_expression)
{
try
{
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression); var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
var rv = exp.getView(new Dictionary<string, StatementExpression>(0)); var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
return rv; return rv;
} catch (Exception ex) { }
catch (Exception ex)
{
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}"); bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
return null; return null;
} }
} }
static void start_echo_client() {
Console.WriteLine("Starting DSPublisher in TCP client echo mode, Hit key to continue");
Console.ReadLine();
SimpleTCPClient client = new SimpleTCPClient("127.0.0.1", 12345);
client.start();
while (true) {
System.Threading.Thread.Sleep(1000);
}
// await SimpleTCPClient.start("127.0.0.1", 12345);
// Task.Delay(1000000000).Wait();
}
void _publish_streams() { void _publish_streams()
{
PubTableBroker pb_broker = new PubTableBroker(); PubTableBroker pb_broker = new PubTableBroker();
CloudStreamManager csm = new CloudStreamManager(); CloudStreamManager csm = new CloudStreamManager();
string csv_folder = @"C:\Temp\csvs_topublish";
CSVPublisher csv_publisher = new CSVPublisher(csv_folder, pb_broker);
// get it from config // get it from config
Dictionary<string, string> streams_to_publish = new Dictionary<string, string>() { Dictionary<string, string> streams_to_publish = new Dictionary<string, string>() {
//{"bbo","bbo()"}, {"bbo","bbo()"},
//{"instruments","instruments()"}, {"instruments","instruments()"},
//{"timesales","timesales(10000)"}, //{"timesales","timesales(10000)"},
//{"books","books()"}, {"books","books()"},
{"seconds_1_10","second(1,10)"}, // {"seconds_1_10","second(1,10)"},
{"literal_420","literal(\"x\", \"420.0\")"}, {"literal_420","literal(\"x\", \"420.0\")"},
{"tradebar_30","tradebars(30)"}, // {"bbo_ts","timeSeries(bbo())"}
{"mongodb_testcollection", "mongodb(\"mongodb://192.168.1.80:27017\",\"local\",\"test_collection\")"} };
}; foreach (var kv in streams_to_publish)
// {"bbo_ts","timeSeries(bbo())"} {
foreach (var kv in streams_to_publish) {
var name = kv.Key; var name = kv.Key;
var exp = kv.Value; var exp = kv.Value;
BCG.ViewStream vs = build_stream(exp); BCG.ViewStream vs = build_stream(exp);
ViewPublisher vpub = new ViewPublisher(name, pb_broker, vs); ViewPublisher vpub = new ViewPublisher(name, pb_broker, vs);
vs.onChange += on_change; vs.onChange += on_change;
} }
while (true) { while (true)
{
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1)); System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
ViewStreamDatabase.instance.doWork(); ViewStreamDatabase.instance.doWork();
csm.doWork(); csm.doWork();
MasterWorker.getInstance(false).doWork();
} }
} }
void on_change(object obj) { void on_change(object obj)
{
bts.Log.Gen.info($"DSPublisher on_change dp: {obj}"); bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
}
public class CSVPublisher : IWorker {
private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
// read from config
private readonly string _csv_directory = @"C:\Temp\csvs_topublish\";
private PubTableBroker _pub_table_broker;
public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) {
_csv_directory = csv_directory;
_pub_table_broker = pub_table_broker;
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
}
public void doWork() {
var file_names = Directory.GetFiles(_csv_directory);
foreach (var fn in file_names) {
if (!_pubs.ContainsKey(fn)) {
_handle_new_file(fn);
}
}
}
private void _handle_new_file(string file_name) {
if (file_name.EndsWith(".csv")) {
ViewStream vs = new CSVReaderStream(file_name);
vs = new SchemaGuesserStream(vs);
var toks = file_name.Split('\\');
string table_name = toks.Last();
table_name = table_name.Remove(table_name.Length - ".csv".Length);
ViewPublisher vp = new ViewPublisher(table_name, _pub_table_broker, vs);
_pubs.Add(file_name, vp);
}
}
} }
@@ -382,7 +382,7 @@ namespace BCG {
private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e) { private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e) {
myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length; myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length;
myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length; myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length;
myQueryTextBox.SelectionColor = Color.White; myQueryTextBox.SelectionColor = Color.Black;
myQueryTextBox.SelectionLength = 0; myQueryTextBox.SelectionLength = 0;
myTryExpression(myQueryTextBox.Text); myTryExpression(myQueryTextBox.Text);
} }
+1 -13
View File
@@ -1,4 +1,4 @@
Microsoft Visual Studio Solution File, Format Version 12.00 Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17 # Visual Studio Version 17
VisualStudioVersion = 17.0.31903.59 VisualStudioVersion = 17.0.31903.59
MinimumVisualStudioVersion = 10.0.40219.1 MinimumVisualStudioVersion = 10.0.40219.1
@@ -38,10 +38,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamTests", "DataStre
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "SharedStreams\SharedStreams.csproj", "{C1EBC423-4668-43D4-B7E5-A291086BA823}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "SharedStreams\SharedStreams.csproj", "{C1EBC423-4668-43D4-B7E5-A291086BA823}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{134C3915-CB2B-494C-BD62-E0B18250CBF5}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}"
EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU Debug|Any CPU = Debug|Any CPU
@@ -123,13 +119,5 @@ Global
{C1EBC423-4668-43D4-B7E5-A291086BA823}.Debug|Any CPU.Build.0 = Debug|Any CPU {C1EBC423-4668-43D4-B7E5-A291086BA823}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C1EBC423-4668-43D4-B7E5-A291086BA823}.Release|Any CPU.ActiveCfg = Release|Any CPU {C1EBC423-4668-43D4-B7E5-A291086BA823}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C1EBC423-4668-43D4-B7E5-A291086BA823}.Release|Any CPU.Build.0 = Release|Any CPU {C1EBC423-4668-43D4-B7E5-A291086BA823}.Release|Any CPU.Build.0 = Release|Any CPU
{134C3915-CB2B-494C-BD62-E0B18250CBF5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{134C3915-CB2B-494C-BD62-E0B18250CBF5}.Debug|Any CPU.Build.0 = Debug|Any CPU
{134C3915-CB2B-494C-BD62-E0B18250CBF5}.Release|Any CPU.ActiveCfg = Release|Any CPU
{134C3915-CB2B-494C-BD62-E0B18250CBF5}.Release|Any CPU.Build.0 = Release|Any CPU
{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection EndGlobalSection
EndGlobal EndGlobal
@@ -47,30 +47,30 @@ namespace BCG {
/// of Int32.Max would return all the messages it has /// of Int32.Max would return all the messages it has
/// received between last getUpdates and this call. /// received between last getUpdates and this call.
/// </summary /// </summary
IList<byte[]> get_updates(TimeSpan timeout, int maxElements); IList<byte[]> getUpdates(TimeSpan timeout, int maxElements);
/// <summary> /// <summary>
/// server URL to which the connection is made /// server URL to which the connection is made
/// </summary> /// </summary>
string server_url { get; } string serverURL { get; }
/// <summary> /// <summary>
/// client URL as seen by the server. if a connection has not /// client URL as seen by the server. if a connection has not
/// been established (as can happen in TCP) an empty string /// been established (as can happen in TCP) an empty string
/// is returned /// is returned
/// </summary> /// </summary>
string client_url { get; } string clientURL { get; }
/// <summary> /// <summary>
/// registers the callback to receive /// registers the callback to receive
/// connection notifications /// connection notifications
/// </summary> /// </summary>
void register_connection_callback(PubSubServerConnectionStatusChange del); void registerCallback(ConnectionStatusChange del);
/// <summary> /// <summary>
/// registers callbacks to reveive data /// registers callbacks to reveive data
/// </summary> /// </summary>
/// <param name="del"></param> /// <param name="del"></param>
void register_data_callback(PubSubServerDataDelegate del); void registerDataCallback(DataDelegate del);
void register_send_confirmation_callback(SendConfirmationDelegate del); void registerSendConfirmationCallback(SendConfirmationDelegate del);
} }
} }
@@ -8,7 +8,6 @@ using System.IO;
//using BCG.Utilities.General; //using BCG.Utilities.General;
using System.Threading; using System.Threading;
using mbts; using mbts;
using bts;
namespace BCG { namespace BCG {
/// <summary> /// <summary>
@@ -21,27 +20,26 @@ namespace BCG {
/// ITCPConnectionHandler.onConnection will be called on any arbitrary thread /// ITCPConnectionHandler.onConnection will be called on any arbitrary thread
/// ///
/// </summary> /// </summary>
public class PubSubServerManagedTCPConnection : IPubSubServerConnection,// external facing interface to communicate msgs to Pubsubserver public class PubSubServerManagedTCPConnection : IPubSubServerConnection,
ITCPMessageHandler, // to get messages from ITCPCOnnection ITCPMessageHandler,
ITCPConnectionHandler { // to get connections from AsyncTCPConnector_APM ITCPConnectionHandler {
private ITCPConnection _connection, _new_connection; private ITCPConnection _connection, _new_connection;
private readonly BlutoSubAddress _server_address; private readonly BlutoSubAddress _server_address;
// if a message(with confirm) can not be sent , its id is queued here, // if a message(with confirm) can not be sent , its id is queued here,
// on reconnection, _on_send_onfirm_callback is fired with a negative on these ids, so the user // on reconnection, _on_send_onfirm_callback is fired with a negative on these ids, so the user
//can re-send it. The re-send mechanism is not in implemented in this class, but by the user of this class //can re-send it. The re-send mechanism is not in implemented in this class, but by the user of this class
private ThreadSafeQueue<string> _negative_unsent_confs = new ThreadSafeQueue<string>(); private ThreadSafeQueue<string> _negative_unsent_confs = new ThreadSafeQueue<string>();
// in onData, data is queued here, for polling // in onData, data is queued here
private SyncQueue<byte[]> _in_data_queue = new SyncQueue<byte[]>(); private SyncQueue<byte[]> _in_data_queue = new SyncQueue<byte[]>();
private DateTime _last_hb = DateTime.Now; private DateTime _last_hb = DateTime.Now;
private PubSubServerConnectionStatusChange _conn_callback; private ConnectionStatusChange _conn_callback;
private PubSubServerDataDelegate _data_callback; private DataDelegate _data_callback;
private SendConfirmationDelegate _on_send_confirm_callback; private SendConfirmationDelegate _on_send_onfirm_callback;
private readonly TimeSpan _connection_drop_threshold = private readonly TimeSpan _connection_drop_threshold =
new TimeSpan(0, 0, 0, 0, new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval * PubSub3Config.instance.PS3_MissingHeartBeatsForGC);
PubSub3Config.instance.PS3_HeartBeatInterval * PubSub3Config.instance.PS3_MissingHeartBeatsForGC);
public PubSubServerManagedTCPConnection(string server_url) public PubSubServerManagedTCPConnection(string serverURL)
: this(new BlutoSubAddress(server_url)) { } : this(new BlutoSubAddress(serverURL)) { }
public PubSubServerManagedTCPConnection(BlutoSubAddress server_address) { public PubSubServerManagedTCPConnection(BlutoSubAddress server_address) {
_server_address = server_address; _server_address = server_address;
@@ -50,17 +48,17 @@ namespace BCG {
private void _connect_to_ps_server() { private void _connect_to_ps_server() {
bts.Log.Gen.info($"PubSubServerManagedTCPConnection._connect_to_ps_server server: {_server_address}"); bts.Log.Gen.info($"PubSubServerManagedTCPConnection._connect_to_ps_server server: {_server_address}");
AsyncTCPConnector_APM.connect_to(_server_address, this); ClientToServerManagedTCPConnector.connect_to(_server_address, this, TimeSpan.FromSeconds(1));
} }
#region IPubSubServerConnection Members #region IPubSubServerConnection Members
public void send_with_confirmation(byte[] data, ref string conf_id) { public void sendWithConfirmation(byte[] data, ref string conf_id) {
bts.Log.Gen.info($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} data: {data.Length} _connection : {_connection != null}"); bts.Log.Gen.info($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} data: {data.Length} myConnection : {_connection != null}");
if (_connection != null) { if (_connection != null) {
_connection.sendWithConfirmation(data, ref conf_id, (msg_id, confirmed) => { _connection.sendWithConfirmation(data, ref conf_id, (msg_id, confirmed) => {
if (_on_send_confirm_callback != null) if (_on_send_onfirm_callback != null)
_on_send_confirm_callback(msg_id, confirmed); _on_send_onfirm_callback(msg_id, confirmed);
}); });
} else { } else {
bts.Log.Gen.warn($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} no connection, buffering send"); bts.Log.Gen.warn($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} no connection, buffering send");
@@ -88,18 +86,18 @@ namespace BCG {
public void send_with_confirmation(PubSubMessage msg, ref string conf_id) { public void send_with_confirmation(PubSubMessage msg, ref string conf_id) {
var bytes = msg.to_bytes(); var bytes = msg.to_bytes();
send_with_confirmation(bytes, ref conf_id); sendWithConfirmation(bytes, ref conf_id);
} }
public IList<byte[]> get_updates(TimeSpan timeout, int maxElements) { public IList<byte[]> getUpdates(TimeSpan timeout, int maxElements) {
return _in_data_queue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements); return _in_data_queue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements);
} }
public string server_url { public string serverURL {
get { return _server_address.ToString(); } get { return _server_address.ToString(); }
} }
public string client_url { public string clientURL {
get { get {
string rv = ""; string rv = "";
lock (this) { lock (this) {
@@ -120,16 +118,16 @@ namespace BCG {
} }
} }
public void register_connection_callback(PubSubServerConnectionStatusChange del) { public void registerCallback(ConnectionStatusChange del) {
_conn_callback = del; _conn_callback = del;
} }
public void register_data_callback(PubSubServerDataDelegate del) { public void registerDataCallback(DataDelegate del) {
_data_callback = del; _data_callback = del;
} }
public void register_send_confirmation_callback(SendConfirmationDelegate del) { public void registerSendConfirmationCallback(SendConfirmationDelegate del) {
_on_send_confirm_callback = del; _on_send_onfirm_callback = del;
} }
#endregion #endregion
@@ -187,6 +185,18 @@ namespace BCG {
break; break;
} }
// this stupid shit, caused duplicating og messages and 2 days of wild goose chase
// if (message.messageType == RemoteMessageType.Data)
// {
// if (_data_callback != null)
// _data_callback(message.remoteConnectionInfo.ToString(), message.message);
// else
// _in_data_queue.enqueue(message.message);
// }
// else
// {
// _last_hb = DateTime.Now;
// }
} }
#endregion #endregion
@@ -194,7 +204,8 @@ namespace BCG {
#region ITCPConnectionHandler Members #region ITCPConnectionHandler Members
/// <summary> /// <summary>
/// will be called on ana rbitrary .Net thread /// This function is called on the main thread (as all other functions
/// except onData)
/// </summary> /// </summary>
/// <param name="connection"></param> /// <param name="connection"></param>
public void onConnection(ITCPConnection connection) { public void onConnection(ITCPConnection connection) {
@@ -235,10 +246,10 @@ namespace BCG {
_connection.register(this); _connection.register(this);
if (_conn_callback != null) if (_conn_callback != null)
_conn_callback(_server_address.ToString(), true); _conn_callback(_server_address.ToString(), true);
if (_on_send_confirm_callback != null) { if (_on_send_onfirm_callback != null) {
var msg_ids = _negative_unsent_confs.dequeueAll(); var msg_ids = _negative_unsent_confs.dequeueAll();
foreach (var id in msg_ids) foreach (var id in msg_ids)
_on_send_confirm_callback(id, false); _on_send_onfirm_callback(id, false);
} }
} }
} }
@@ -253,6 +264,47 @@ namespace BCG {
public void doWork() { public void doWork() {
//_dispose_and_reconnect_if_missing_hb(); //_dispose_and_reconnect_if_missing_hb();
_check_for_new_connection(); _check_for_new_connection();
///check for heartbeats and let clients know in
///case of missing hb (connection breaking)
// TimeSpan hb_interval = DateTime.Now - myLastHB;
// bool hb_missed = hb_interval > myConnectionDropThreshold;
// bool conn_disposed = myConnection != null || myConnection.isDisposed;
// if (conn_disposed || hb_missed)
// {
// if (hb_missed)
// {
// // since conn breakage was diagnosed logically, lets physically close the connection
// myConnection.Dispose();
// string dispose_msg = $"PubSubServerManagedTCPConnection.doWork server: {myServerAddress} Disposing connection conn_disposed: {conn_disposed} hb_missed: {hb_missed}";
// Console.WriteLine(dispose_msg);
// }
// myConnection = null; // this is used as a marker for a dead connection
// if (myConnCallback != null)
// myConnCallback(myServerAddress.ToString(), false);
// string reconn_msg = $"PubSubServerManagedTCPConnection.doWork disconnecting client from server: {myServerAddress} conn_disposed: {conn_disposed} hb_missed: {hb_missed} " +
// $"because of missing hbs last_hb: {myLastHB}, now: {DateTime.Now} curr_hb_interval: {hb_interval.TotalSeconds} threshold: {myConnectionDropThreshold.TotalSeconds} conn_disposed: {conn_disposed}";
// ClientToServerManagedTCPConnector.connectTo(myServerAddress, this);
// }
// if (myTempNewConnection != null)
// {
// if (myConnection != null)
// {
// // TODOds
// // log error and close it
// // BlueEventLog.logAssert(myConnection == null, "Connection has to be nulled out and disposed "
// // + " before a new connection is established");
// }
// Console.WriteLine("PubSubServerManagedTCPConnection hooking new connection");
// myConnection = myTempNewConnection;
// myTempNewConnection = null;
// myConnection.register(this);
// myConfs.dequeueAll();
// if (myConnCallback != null)
// myConnCallback(myServerAddress.ToString(), true);
// }
// foreach (Pair<string, bool> p in myConfs.dequeueAll())
// if (myOnSendConfirmCallback != null)
// myOnSendConfirmCallback(p.first, p.second);
} }
#endregion #endregion
@@ -16,7 +16,6 @@ namespace BCG {
// SubMetaData // SubMetaData
// TransactionMetaData // TransactionMetaData
// UnSubMetaData // UnSubMetaData
//
public class MetaData : ICloneable { public class MetaData : ICloneable {
private IDictionary _name_value_map = new Dictionary<string, string>(); private IDictionary _name_value_map = new Dictionary<string, string>();
int mySize = -1; int mySize = -1;
@@ -1,8 +1,10 @@
using System; using System;
namespace BCG { namespace BCG
//[Serializable] {
public class SubMetaData { [Serializable]
public class SubMetaData
{
/// <summary> /// <summary>
/// connection URL on which the connection will receive data /// connection URL on which the connection will receive data
/// </summary> /// </summary>
@@ -11,18 +13,20 @@ namespace BCG {
/// identification of the client and the connection on which to receive data /// identification of the client and the connection on which to receive data
/// machine; ip; port; clientURL /// machine; ip; port; clientURL
/// </summary> /// </summary>
public readonly string subscribing_app = ClientInfo.thisClient.processString; public string subscribingApp;
/// <summary> /// <summary>
/// tags that are to be subscribed /// tags that are to be subscribed
/// </summary> /// </summary>
public string[] tags; public string[] tags;
public SubMetaData() { } public SubMetaData()
{ }
public SubMetaData(string the_connection, params string[] the_tags) { public SubMetaData(string theConnection, params string[] theTags)
connection = the_connection; {
tags = the_tags; connection = theConnection;
//subscribing_app = ClientInfo.thisClient.processString; tags = theTags;
subscribingApp = ClientInfo.thisClient.processString;
} }
} }
} }
@@ -1,17 +1,21 @@
using System; using System;
namespace BCG { namespace BCG
//[Serializable] {
public class UnSubMetaData { [Serializable]
public class UnSubMetaData
{
public string connection; public string connection;
public readonly string[] tags; public string[] tags;
public readonly string unsubscribing_app = ClientInfo.thisClient.processString; public string unsubscribingApp = ClientInfo.thisClient.processString;
public UnSubMetaData() { } public UnSubMetaData()
{}
public UnSubMetaData(string the_connection, params string[] the_tags) { public UnSubMetaData(string theConnection, params string[] theTags)
connection = the_connection; {
tags = the_tags; connection = theConnection;
tags = theTags;
} }
} }
} }
File diff suppressed because it is too large. Load diff
@@ -1,248 +1,296 @@
// using BCG; using BCG;
// using Google.Protobuf; using Google.Protobuf;
// using System; using System;
// using System.IO; using System.IO;
// using System.Linq; using System.Linq;
// using System.Net; using System.Net;
// using System.Net.Sockets; using System.Net.Sockets;
// using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
// using System.Text; using System.Text;
// using System.Threading.Tasks; using System.Threading.Tasks;
// using BCG; using BCG;
// using System.Xml.Serialization; using System.Xml.Serialization;
// using System.Reflection; using System.Reflection;
// using BCG.Utilities.General; using BCG.Utilities.General;
// using System.Net.WebSockets; using System.Net.WebSockets;
// namespace BCG { namespace BCG
{
// public class AsyncTCPConnection2 : ITCPConnection, IWorker, IDisposable { public class AsyncTCPConnection2 : ITCPConnection, IWorker, IDisposable
// private Socket _socket; {
// // tied to the socket above //private Socket _socket;
// private NetworkStream _nw_stream; // tied to the socket above
// private readonly ConnectionSide _side; private NetworkStream _nw_stream;
// ITCPMessageHandler _message_handler; private readonly ConnectionSide _side;
// private int _msg_id = 0; ITCPMessageHandler _message_handler;
// RemoteConnectionInfo _peer_address; private int _msg_id = 0;
// RemoteConnectionInfo _self_address; RemoteConnectionInfo _peer_address;
// private TimeSpan _hb_period = TimeSpan.FromSeconds(1); RemoteConnectionInfo _self_address;
// private DateTime _last_hb_sent = DateTime.MinValue; private TimeSpan _hb_period = TimeSpan.FromSeconds(1);
// private DateTime _last_hb_received = DateTime.MaxValue; private DateTime _last_hb_sent = DateTime.MinValue;
// private static byte _msg_type_data = 1; private DateTime _last_hb_received = DateTime.MaxValue;
// private static byte _msg_type_hb = 0; private static byte _msg_type_data = 1;
// private bool _is_disposed = false; private static byte _msg_type_hb = 0;
// SyncQueue<QueuedMessage> _data_send_queue = new SyncQueue<QueuedMessage>(); private bool _is_disposed = false;
SyncQueue<QueuedMessage> _data_send_queue = new SyncQueue<QueuedMessage>();
// public class QueuedMessage { public class QueuedMessage
// public string id; {
// public SendConfirmationDelegate del; public string id;
// public byte[] data; public SendConfirmationDelegate del;
public byte[] data;
// public QueuedMessage(string id, SendConfirmationDelegate del, byte[] data) { public QueuedMessage(string id, SendConfirmationDelegate del, byte[] data)
// id = id; {
// del = del; id = id;
// data = data; del = del;
// } data = data;
}
// public QueuedMessage(byte[] data) public QueuedMessage(byte[] data)
// : this(String.Empty, null, data) { } : this(String.Empty, null, data) { }
// } }
// public enum ConnectionSide { public enum ConnectionSide
// Client, {
// Server Client,
// } Server
}
// public AsyncTCPConnection2(Socket socket, ConnectionSide side) { public AsyncTCPConnection2(Socket socket, ConnectionSide side)
// _socket = socket; {
// _nw_stream = new NetworkStream(socket); //_socket = socket;
// _side = side; _nw_stream = new NetworkStream(socket);
// IPEndPoint r_ep = socket.RemoteEndPoint as IPEndPoint; _side = side;
// _peer_address = new RemoteConnectionInfo(r_ep.Address.ToString(), r_ep.Port); IPEndPoint r_ep = socket.RemoteEndPoint as IPEndPoint;
_peer_address = new RemoteConnectionInfo(r_ep.Address.ToString(), r_ep.Port);
// var loc_ep = socket.LocalEndPoint as IPEndPoint; var loc_ep = socket.LocalEndPoint as IPEndPoint;
// _self_address = new RemoteConnectionInfo(loc_ep.Address.ToString(), loc_ep.Port); _self_address = new RemoteConnectionInfo(loc_ep.Address.ToString(), loc_ep.Port);
// } }
// public void register(ITCPMessageHandler message_handler) { public void register(ITCPMessageHandler message_handler)
// _message_handler = message_handler; {
// Task.Run(async () => _keep_reading()); _message_handler = message_handler;
// Task.Run(async () => await _keep_sending()); Task.Run(async () => _keep_reading());
// //_keep_reading(); Task.Run(async () => await _keep_sending());
// // if add a _keep_sending threadis added then we can maek send a non async //_keep_reading();
// // methid making it much easier to integrate with the rest // if add a _keep_sending threadis added then we can maek send a non async
// } // methid making it much easier to integrate with the rest
}
// // this thread pulls // this thread pulls
// private async Task _keep_sending() { private async Task _keep_sending()
// bts.Log.Gen.info($"TCPConnection._keep_sending starting"); {
// while (!_is_disposed) { bts.Log.Gen.info($"TCPConnection._keep_sending starting");
// try { while (!_is_disposed)
// QueuedMessage m = _data_send_queue.dequeue(100); {
// if (m == null) { try
// continue; {
// } QueuedMessage m = _data_send_queue.dequeue(100);
// await send_async(m.data); if (m == null)
// if (String.IsNullOrEmpty(m.id)) { {
// m.del(m.id, true); continue;
// } }
// } catch (Exception ex) { await send_async(m.data);
// bts.Log.Gen.error($"TCPConnection._keep_sending ex: {ex.Message}"); if (String.IsNullOrEmpty(m.id))
// await Task.Delay(100); {
// _handle_error(ex); m.del(m.id, true);
// } }
// } }
// } catch (Exception ex)
{
bts.Log.Gen.error($"TCPConnection._keep_sending ex: {ex.Message}");
await Task.Delay(100);
_handle_error(ex);
}
}
}
// private async Task _keep_reading() { private async Task _keep_reading()
// bts.Log.Gen.info($"TCPConnection._keep_reading starting"); {
// while (!_is_disposed) { bts.Log.Gen.info($"TCPConnection._keep_reading starting");
// try { while (!_is_disposed)
// byte[] msg = await TCPSocketUtils.read_bytes_async(_nw_stream); {
// bts.Log.Gen.info($"AsyncTCPConnection2._keep_reading side: {_side} self: {get_self_address()} peer: {get_peer_address()} msg len: {msg.Length}"); try
// await _handle_msg(msg); {
// } catch (Exception ex) { byte[] msg = await TCPSocketUtils.read_bytes_async(_nw_stream);
// bts.Log.Gen.error($"TCPConnection._keep_reading ex: {ex.Message}"); bts.Log.Gen.info($"AsyncTCPConnection2._keep_reading side: {_side} self: {get_self_address()} peer: {get_peer_address()} msg len: {msg.Length}");
// await Task.Delay(100); await _handle_msg(msg);
// _handle_error(ex); }
// } catch (Exception ex)
// } {
// } bts.Log.Gen.error($"TCPConnection._keep_reading ex: {ex.Message}");
await Task.Delay(100);
_handle_error(ex);
}
}
}
// private async Task _handle_msg(byte[] msg) { private async Task _handle_msg(byte[] msg)
// // read the first byte to read the msg type and deal with it accordingly {
// BinaryReader br = new BinaryReader(new MemoryStream(msg)); // read the first byte to read the msg type and deal with it accordingly
// byte msg_type = br.ReadByte(); BinaryReader br = new BinaryReader(new MemoryStream(msg));
// if (msg_type == _msg_type_data) { byte msg_type = br.ReadByte();
// byte[] data = br.ReadBytes(msg.Length - 1); if (msg_type == _msg_type_data)
// RemoteMessage m = new RemoteMessage(data, RemoteMessageType.Data, _peer_address); {
// _message_handler.onData(m); byte[] data = br.ReadBytes(msg.Length - 1);
// } else { RemoteMessage m = new RemoteMessage(data, RemoteMessageType.Data, _peer_address);
// RemoteMessage m = new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_address); _message_handler.onData(m);
// _message_handler.onData(m); }
// } else
// } {
RemoteMessage m = new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_address);
_message_handler.onData(m);
}
}
// // not very efficient // not very efficient
// private static byte[] _prefix_msg_type(byte[] data) { private static byte[] _prefix_msg_type(byte[] data)
// MemoryStream ms = new MemoryStream(); {
// BinaryWriter bw = new BinaryWriter(ms); MemoryStream ms = new MemoryStream();
// bw.Write(_msg_type_data); BinaryWriter bw = new BinaryWriter(ms);
// bw.Write(data); bw.Write(_msg_type_data);
// ms.Position = 0; bw.Write(data);
// return ms.ToArray(); ms.Position = 0;
// } return ms.ToArray();
}
// private async void _handle_error(Exception ex) { private async void _handle_error(Exception ex)
// bts.Log.Gen.error($"TCPConnection._handle_error ex: {ex}"); {
// try { bts.Log.Gen.error($"TCPConnection._handle_error ex: {ex}");
// var socket = _nw_stream.Socket; try
// socket.Close(); {
// socket.Dispose(); var socket = _nw_stream.Socket;
// } catch (Exception ex2) { socket.Close();
// bts.Log.Gen.error($"TCPConnection._handle_error got exception when closing Socket ex: {ex}"); socket.Dispose();
// } }
// _is_disposed = true; catch (Exception ex2)
// RemoteMessage rm = new RemoteMessage(new byte[] { }, RemoteMessageType.Disconnection, _peer_address); {
// _message_handler.onData(rm); bts.Log.Gen.error($"TCPConnection._handle_error got exception when closing Socket ex: {ex}");
// } }
_is_disposed = true;
RemoteMessage rm = new RemoteMessage(new byte[] { }, RemoteMessageType.Disconnection, _peer_address);
_message_handler.onData(rm);
}
// public void send_with_confirmation(byte[] data, string confId, SendConfirmationDelegate del) { } public void send_with_confirmation(byte[] data, string confId, SendConfirmationDelegate del) { }
// public async Task send_with_confirmation_async(byte[] data, string conf_id, SendConfirmationDelegate del) { public async Task send_with_confirmation_async(byte[] data, string conf_id, SendConfirmationDelegate del)
// try { {
// conf_id = (++_msg_id).ToString(); try
// await TCPSocketUtils.send_bytes_async(_nw_stream, data); {
// del(conf_id, true); conf_id = (++_msg_id).ToString();
// } catch (Exception ex) { await TCPSocketUtils.send_bytes_async(_nw_stream, data);
// _handle_error(ex); del(conf_id, true);
// } }
// } catch (Exception ex)
{
_handle_error(ex);
}
}
// public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del) { public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del)
// conf_id = (++_msg_id).ToString(); {
// send(data); conf_id = (++_msg_id).ToString();
// del(conf_id, true); send(data);
// } del(conf_id, true);
}
// public void send(byte[] data) { public void send(byte[] data)
// Task.Run(async () => await send_async(data)); {
// bts.Log.Gen.info($"AsyncTCPConnection2.send self: {get_self_address()} peer: {get_peer_address()} msg len: {data.Length} sent successfully"); Task.Run(async () => await send_async(data));
// //_data_send_queue.enqueue(new QueuedMessage(data)); bts.Log.Gen.info($"AsyncTCPConnection2.send self: {get_self_address()} peer: {get_peer_address()} msg len: {data.Length} sent successfully");
// } //_data_send_queue.enqueue(new QueuedMessage(data));
}
// public async Task send_async(byte[] data) { public async Task send_async(byte[] data)
// // prefix 4 bytes for type (data vs hb) {
// MemoryStream str = new MemoryStream(); // prefix 4 bytes for type (data vs hb)
// BinaryWriter bw = new BinaryWriter(str); MemoryStream str = new MemoryStream();
// bw.Write(_msg_type_data); BinaryWriter bw = new BinaryWriter(str);
// bw.Write(data); bw.Write(_msg_type_data);
// try { bw.Write(data);
// await TCPSocketUtils.send_bytes_async(_nw_stream, str.ToArray()); try
// } catch (Exception ex) { {
// bts.Log.Gen.error($"AsyncTCPConnection2.send_async ex: {ex}"); await TCPSocketUtils.send_bytes_async(_nw_stream, str.ToArray());
// _handle_error(ex); }
// } catch (Exception ex)
// } {
bts.Log.Gen.error($"AsyncTCPConnection2.send_async ex: {ex}");
_handle_error(ex);
}
}
// public void getPeerAddress(out string ip, out int port) { public void getPeerAddress(out string ip, out int port)
// ip = _peer_address.ip; {
// port = _peer_address.port; ip = _peer_address.ip;
// ///.Net has some problems getting the port, so using port = _peer_address.port;
// ///native code for that ///.Net has some problems getting the port, so using
// //sockaddr addr = new sockaddr(); ///native code for that
// //int namelen = 16; //sockaddr addr = new sockaddr();
// //getsockname(mySock.Handle, ref addr, ref namelen); //int namelen = 16;
// //port = addr.sin_port; //getsockname(mySock.Handle, ref addr, ref namelen);
// } //port = addr.sin_port;
}
// public RemoteConnectionInfo get_peer_address() { public RemoteConnectionInfo get_peer_address()
// string ip; int port; {
// getPeerAddress(out ip, out port); string ip; int port;
// return new RemoteConnectionInfo(ip, port); getPeerAddress(out ip, out port);
// } return new RemoteConnectionInfo(ip, port);
}
// public RemoteConnectionInfo get_self_address() { public RemoteConnectionInfo get_self_address()
// string ip; int port; {
// getSelfAddress(out ip, out port); string ip; int port;
// return new RemoteConnectionInfo(ip, port); getSelfAddress(out ip, out port);
// } return new RemoteConnectionInfo(ip, port);
}
// public void getSelfAddress(out string ip, out int port) { public void getSelfAddress(out string ip, out int port)
// ip = _self_address.ip; {
// port = _self_address.port; ip = _self_address.ip;
// } port = _self_address.port;
}
// private async void _send_hb() { private async void _send_hb()
// if (_last_hb_sent - DateTime.Now > _hb_period) { {
// MemoryStream ms = new MemoryStream(); if (_last_hb_sent - DateTime.Now > _hb_period)
// BinaryWriter bw = new BinaryWriter(ms); {
// bw.Write(_msg_type_hb); MemoryStream ms = new MemoryStream();
// ms.Position = 0; BinaryWriter bw = new BinaryWriter(ms);
// byte[] hb_msg = ms.ToArray(); bw.Write(_msg_type_hb);
// await TCPSocketUtils.send_bytes_async(_nw_stream, hb_msg); ms.Position = 0;
// } byte[] hb_msg = ms.ToArray();
// } await TCPSocketUtils.send_bytes_async(_nw_stream, hb_msg);
}
}
// public void doWork() { public void doWork()
// _send_hb(); {
// } _send_hb();
}
// public void Dispose() { public void Dispose()
// _nw_stream.Close(); {
// _nw_stream.Dispose(); _nw_stream.Close();
// _is_disposed = true; _nw_stream.Dispose();
// } _is_disposed = true;
}
// public bool isDisposed { public bool isDisposed
// get { {
// return _is_disposed; get
// } {
// } return _is_disposed;
}
}
// } }
// } }
@@ -1,75 +0,0 @@
using System.Net.Sockets;
using System.Net;
using BCG;
using System;
using System.Collections.Generic;
namespace bts {
// opens a TCP port and starts listening to connections
public class AsyncTCPConnectionAcceptor_APM {
private readonly int _port;
private Socket _accept_socket;
private ITCPConnectionHandler _connection_handler;
private HashSet<string> _key_map = new HashSet<string>();
public AsyncTCPConnectionAcceptor_APM(int port, ITCPConnectionHandler connection_handler) {
_port = port;
_connection_handler = connection_handler;
}
public void init() {
_init();
_begin_accepting();
}
private void _init() {
_accept_socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
IPEndPoint endpoint = new IPEndPoint(IPAddress.Any, _port);
_accept_socket.Bind(endpoint);
_accept_socket.Listen(250);
}
private void _begin_accepting() {
var accept_key = Guid.NewGuid().ToString();
var res = _accept_socket.BeginAccept(new AsyncCallback(_on_accept), accept_key);
if (res != null && res.IsCompleted && res.CompletedSynchronously)
_on_accept(res);
}
private void _on_accept(IAsyncResult res) {
bts.Log.Gen.info("AsyncTCPConnectionAcceptor_APM._on_accept state: {res.AsyncState}");
string key = res.AsyncState.ToString();
if (_key_map.Contains(key)) {
return;
}
_key_map.Add(key);
Socket new_sock;
try {
new_sock = _accept_socket.EndAccept(res);
} catch (Exception ex) {
bts.Log.Gen.info("AsyncTCPConnectionAcceptor_APM._on_accept exception thrown by EndAccept ex: {ex}");
System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(250));
_begin_accepting();
return;
}
new_sock.LingerState = new LingerOption(false, 0);
new_sock.Blocking = false;
ITCPConnection newConn = new AsyncTCPConnection_APM(new_sock, TimeSpan.FromSeconds(1), ITCPConnection.ConnectionSide.Server);
_connection_handler.onConnection(newConn);
// go back to accepting more conns
_begin_accepting();
string ip; int port;
newConn.getPeerAddress(out ip, out port);
string connMsg = String.Format("Connection to port {2} made from {0}:{1}", ip, port, _port);
// BlueEventLog.logEvent(connMsg, PS3LogLevels.connection);
//Console.WriteLine(connMsg);
}
}
}
@@ -1,584 +0,0 @@
//This implementation uses APM the beginFunc, endFunc pattern
// https://learn.microsoft.com/en-us/dotnet/standard/asynchronous-programming-patterns/asynchronous-programming-model-apm#beginning-an-asynchronous-operation
/// <summary>
/// This class, mantains a connection to a peer socket and using the APM
/// model to communicate. It sends heartbeats at regular
/// intervals (config). It is in active state when messages
/// are sent and received without an error state. It is assumed
/// dead state when either of the following happens
/// BeginReceive returns a SocketError
/// BeginSend returns a SocketError
/// EndReceive returns a SocketError
/// EndSend returns a SocketError
/// As soon as it assumes dead state it stops sending or
/// receiving any messages (including heartbeats).
/// The user of this class can either detect that by
/// missing heartbeats or checking the isConnected property.
/// If the peer dies in a 'silent' way, where none of the above
/// methods indicate an error, missing heartbeats is guaranteed
/// to detect it.
/// </summary>
using System;
using System.Collections.Generic;
using System.Text;
using System.Net;
using System.Net.Sockets;
using System.IO;
using BCG.Utilities.General;
using System.Runtime.InteropServices;
using BCG;
using System.Collections.ObjectModel;
using System.Numerics;
using System.Reflection.Metadata;
using System.Threading.Tasks;
using System.Linq;
namespace bts {
public class AsyncTCPConnection_APM : ITCPConnection, IWorker, IDisposable {
// serves as the base class for both Send and ReeceiveMessage classes.
// since both TCP sends and receives happen in multiple transfers, this
// provides a wrapper around copying the partial transfers into the complete buffer
public class AsyncMessageBase {
readonly string msg_id = Guid.NewGuid().ToString();
// every BeginSend or BeginReceive has a unique key associated with it
public int key;
// total msg that needs to be transfered
public byte[] buffer;
// as data is transferred in chunks, transfer_ptr moves ahead by amount of data transferred.
// everything < transfer_ptr has been successfully transferred,
// ie this is pointing to where the next transfer will start from
public int transfer_ptr;
protected int _n_transfers = 0;
public AsyncMessageBase(byte[] the_buffer, int the_offset) {
buffer = the_buffer;
transfer_ptr = the_offset;
}
public int remaining_size {
get {
return buffer.Length - transfer_ptr;
}
}
public int original_size => buffer.Length;
public void advance_transfer_pointer(int advance_by) {
transfer_ptr += advance_by;
_n_transfers++;
if (transfer_ptr > original_size) {
bts.Log.Gen.error($"transfer went beyond initial size {ToString()}");
}
// TODO: log if it goes past the buffer
}
public bool fully_transfered() {
return transfer_ptr >= original_size;
}
public override string ToString() {
return $"{GetType().ToString()} msg_id: {msg_id} key: {key} original_length: {original_size} transfer_ptr: {transfer_ptr} n_transfers: {_n_transfers} " +
$"fully_transfered: {fully_transfered()}";
}
}
public class AsyncSendMessage : AsyncMessageBase {
public string message_id;
public SendConfirmationDelegate on_send_del;
//private int _n_transfers = 0;
public AsyncSendMessage(byte[] the_buffer, int the_offset)
: this(the_buffer, the_offset, String.Empty, null) { }
public AsyncSendMessage(byte[] the_buffer, int the_transfer_ptr, string the_message_id, SendConfirmationDelegate the_on_send_del)
: base(the_buffer, the_transfer_ptr) {
message_id = the_message_id;
on_send_del = the_on_send_del;
}
public override string ToString() {
return base.ToString();
}
}
public class AsyncReceiveMessage : AsyncMessageBase {
public enum ReadState {
reading_size,
reading_data
};
readonly ReadState read_state;
public AsyncReceiveMessage(int size, ReadState the_read_state) :
this(new byte[size], 0, the_read_state) { }
public AsyncReceiveMessage(byte[] the_buffer, int the_transfer_ptr, ReadState the_read_state)
: base(the_buffer, the_transfer_ptr) {
read_state = the_read_state;
}
public int read_int32(int start_idx = 0) {
return BitConverter.ToInt32(buffer, start_idx);
}
public byte[] read_bytes(int start_idx) {
byte[] rv = new byte[buffer.Length - start_idx];
Array.Copy(buffer, start_idx, rv, 0, rv.Length);
return rv;
}
public bool is_reading_size() => read_state == ReadState.reading_size;
}
public class ErrorMessage {
public readonly SocketError? socket_error = null;
public readonly Exception exception = null;
public readonly bool is_exception = false;
public readonly string context = String.Empty;
public ErrorMessage(SocketError socket_error_, string context_) {
socket_error = socket_error_;
context = context_;
}
public ErrorMessage(Exception exception_, string context_) {
is_exception = true;
exception = exception_;
context = context_;
}
public bool oom => exception != null && exception is OutOfMemoryException;
public override string ToString() {
if (is_exception)
return $"ErrorMessage exception: {exception} context: {context}";
return $"socket_error: {socket_error} context: {context}";
}
}
#region work around for .Net socket's LocalEndPoint bug
// [StructLayout(LayoutKind.Sequential)]
// private struct in_addr {
// [MarshalAs(UnmanagedType.ByValArray, SizeConst = 4)]
// public byte[] sin_addr;
// }
// [StructLayout(LayoutKind.Sequential)]
// private struct sockaddr {
// public short sin_family;
// public ushort sin_port;
// public in_addr sin_addr;
// [MarshalAs(UnmanagedType.ByValArray, SizeConst = 8)]
// public byte[] sin_zero;
// }
// [DllImport("wsock32.dll")]
// private static extern int getsockname(IntPtr s, ref sockaddr name, ref int namelen);
// [DllImport("wsock32.dll")]
// private static extern int getpeername(IntPtr s, ref sockaddr name, ref int namelen);
#endregion
private const int DATA_MSG_SIGNAL = 0;
private const int HB_MSG_SIGNAL = 1;
private int _send_key = 0, _receive_key = 0;
private HashSet<int> _read_key_map = new HashSet<int>(),
_send_key_map = new HashSet<int>();
private readonly int _max_msg_size = PubSub3Config.instance.PS3_MaxPacketSize;
private readonly RemoteConnectionInfo _peer_conn_info, _self_conn_info;
private readonly ITCPConnection.ConnectionSide _connection_side;
private readonly Socket _sock;
private ITCPMessageHandler _message_handler = null;
private bool _is_alive = true;
private byte[] _cached_hb_message = null;
private readonly TimeSpan _hb_period = TimeSpan.FromSeconds(1); // TODO get it from config
private MasterWorker _master_worker = MasterWorker.getInstance(true);
private bool _hb_received_yet = false, _data_received_yet = false, _hb_sent_once = false;
private DateTime _last_hb_received_time = new DateTime(1900, 1, 1);
public AsyncTCPConnection_APM(Socket sock, TimeSpan hb_period, ITCPConnection.ConnectionSide connection_side) {
_sock = sock;
_hb_period = hb_period;
_connection_side = connection_side;
_sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.DontRoute, true);
string ip;
int port;
getPeerAddress(out ip, out port);
_peer_conn_info = new RemoteConnectionInfo(ip, port);
getSelfAddress(out ip, out port);
_self_conn_info = new RemoteConnectionInfo(ip, port);
bts.Log.Gen.info($"AsyncTCPConnection_APM created {ToString()}");
}
public override string ToString() {
return $"AsyncTCPConnection_APM side: {_connection_side} peer: {_peer_conn_info}, self: {_self_conn_info}";
}
#region ITCPConnection Members
public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del) {
conf_id = Guid.NewGuid().ToString();
byte[] msg = _build_data_message(data);
_begin_or_resume_sending(new AsyncSendMessage(msg, 0, conf_id, del));
}
public void send(byte[] data) {
byte[] msg = _build_data_message(data);
_begin_or_resume_sending(new AsyncSendMessage(msg, 0));
}
public void register(ITCPMessageHandler messageHandler) {
bts.Log.assert_d(_message_handler == null, "Message Handler is already assigned");
bts.Log.Gen.info($"AsyncTCPConnection_APM for peer = {_peer_conn_info} being registered");
_message_handler = messageHandler;
_master_worker.addWorker(this, _hb_period);
// we send only when send is called but read continoulsy, starting here
_read_next_msg();
}
public void getPeerAddress(out string ip, out int port) {
ip = "";
port = 0;
if (_is_alive) {
IPEndPoint ipEp = _sock.RemoteEndPoint as IPEndPoint;
port = ipEp.Port;
ip = ipEp.Address.ToString();
}
}
public void getSelfAddress(out string ip, out int port) {
ip = "";
port = 0;
if (_is_alive) {
IPEndPoint ipEp = _sock.LocalEndPoint as IPEndPoint;
ip = ipEp.Address.ToString();
port = ipEp.Port;
}
}
public bool isDisposed {
get { return !_is_alive; }
}
#endregion
#region IWorker Members
bool _last_connected = false;
public void doWork() {
if (!_is_alive)
return;
return;
// disable hbs for now
if (!_hb_sent_once) {
bts.Log.Gen.info("Sending heartbeat to peer {_peer_conn_info} for the first time");
_hb_sent_once = true;
}
var msg = _build_hb_message();
_begin_or_resume_sending(new AsyncSendMessage(msg, 0));
}
private static int _tid() {
var tid = System.Threading.Thread.CurrentThread.ManagedThreadId;
return tid;
}
private static string _cb_txt(IAsyncResult res) {
return $"completed?: {res.IsCompleted} sync?: {res.CompletedSynchronously}";
}
#endregion
#region sending data
private void _begin_or_resume_sending(AsyncSendMessage message) {
bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_sending this: {ToString()} tid: {_tid()} message: {message} _is_alive: {_is_alive}");
if (_is_alive) {
try {
SocketError begin_send_err;
// since we get message on on_send we dont need to keep the state around
message.key = _send_key++;
var res = _sock.BeginSend(message.buffer, message.transfer_ptr, message.remaining_size, SocketFlags.None,
out begin_send_err, new AsyncCallback(_on_send), message);
if (begin_send_err != SocketError.Success)
_handle_error(new ErrorMessage(begin_send_err, "_begin_or_resume_sending"));
else {
// successfully sent, all we need to do here is make sure if it was sent asynchronously
// if not call _on_send explicitely
if (res != null && res.IsCompleted && res.CompletedSynchronously) {
// for small messages on LANs this can happen, we will on_send to deal with it the same way
Task.Run(() => {
bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_sending calling _on_send explicitely");
_on_send(res);
});
}
}
} catch (Exception ex) {
_handle_error(new ErrorMessage(ex, "_begin_or_resume_sending"));
}
}
}
private void _on_send(IAsyncResult res) {
AsyncSendMessage message = res.AsyncState as AsyncSendMessage;
if (_send_key_map.Contains(message.key)) {
bts.Log.Gen.warn($"AsyncTCPConnection_APM._on_send key: {message.key} already handled");
return;
}
_send_key_map.Add(message.key);
bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: pre endSend this: {ToString()} tid: {_tid()} " +
$"res: {_cb_txt(res)} message: {res.AsyncState} _is_alive: {_is_alive}");
if (_is_alive) {
try {
SocketError send_err;
int bytes_sent = _sock.EndSend(res, out send_err);
//message.offset += bytesSent;
message.advance_transfer_pointer(bytes_sent);
if (bytes_sent == 0) {
bts.Log.Gen.error($"AsyncTCPConnection_APM._on_send this: {this} sent 0 bytes, throwing exception");
_handle_error(new ErrorMessage(new ApplicationException("Sent 0 bytes, peer socket is likely shutdown"),
"_on_send"));
return;
}
bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: post endSend this: {ToString()} tid: {_tid()} res: {_cb_txt(res)} " +
$"message: {res.AsyncState} bytes_sent: {bytes_sent} _is_alive: {_is_alive}");
if (send_err == SocketError.Success) {
// successful send of some bytes
//if (message.offset == message.buffer.Length) {
if (message.fully_transfered()) {
// fully sent, if confirmation needs to be sent out
if (!String.IsNullOrEmpty(message.message_id)) {
message.on_send_del(message.message_id, true);
}
} else {
// there is more to send
_begin_or_resume_sending(message);
}
} else {
_handle_error(new ErrorMessage(send_err, "_on_send"));
}
} catch (Exception ex) {
_handle_error(new ErrorMessage(ex, "_on_send"));
}
}
}
/// <summary>
/// Assembles the heartbeat message, and uses it for
/// future references
/// </summary>
/// <returns></returns>
private byte[] _build_hb_message() {
if (_cached_hb_message == null) {
BinaryWriter bw = new BinaryWriter(new MemoryStream());
// the total msg =[msg_sz, msg_type]
int msg_sz = 4;
bw.Write(msg_sz);
bw.Write(HB_MSG_SIGNAL);
_cached_hb_message = (bw.BaseStream as MemoryStream).ToArray();
}
return _cached_hb_message;
}
// writes size prefixed msg
private byte[] _build_data_message(byte[] data) {
// TODO, avoid unecessary memory creation (write directly into byte[] using BitConverter)
BinaryWriter bw = new BinaryWriter(new MemoryStream());
// the total msg =[msg_sz, msg_type, data]
var msg_sz = data.Length + 4;
bw.Write(msg_sz); // +4 for the type of msg
// to indicate data vs hb
bw.Write(DATA_MSG_SIGNAL); // for data
// if we want to id every message, this is the place to do it
bw.Write(data);
return (bw.BaseStream as MemoryStream).ToArray();
}
#endregion
#region receiving data
private void _read_next_msg() {
_read_size();
}
private void _read_size() {
_begin_or_resume_reading(new AsyncReceiveMessage(4, AsyncReceiveMessage.ReadState.reading_size));
}
private void _read_data(int data_size) {
// TODO, this could be a problem is the size is too big - maybe read it in chunks (later)
_begin_or_resume_reading(new AsyncReceiveMessage(data_size, AsyncReceiveMessage.ReadState.reading_data));
}
private void _begin_or_resume_reading(AsyncReceiveMessage rcv_msg) {
bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_reading: this: {ToString()} tid: {_tid()} message: {rcv_msg} _is_alive: {_is_alive}");
if (!_is_alive)
return;
try {
SocketError read_sock_err;
rcv_msg.key = _receive_key++;
var res = _sock.BeginReceive(rcv_msg.buffer, rcv_msg.transfer_ptr,
rcv_msg.remaining_size,
SocketFlags.None, out read_sock_err,
new AsyncCallback(_on_read),
rcv_msg);
if (read_sock_err != SocketError.Success) {
bts.Log.Gen.error($"_begin_or_resume_reading this: {this} error: {read_sock_err} rcv_msg: {rcv_msg}");
_handle_error(new ErrorMessage(read_sock_err, "_beging_or_resume_reading"));
} else {
// successfully started receiving
if (res != null && res.IsCompleted && res.CompletedSynchronously) {
// make it follow the same code path
Task.Run(() => {
bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_reading calling _on_send explicitely");
_on_read(res);
}
);
}
}
} catch (Exception ex) {
_handle_error(new ErrorMessage(ex, "_beging_or_resume_reading"));
}
}
private void _on_read(IAsyncResult res) {
AsyncReceiveMessage rcv_message = res.AsyncState as AsyncReceiveMessage;
if (_read_key_map.Contains(rcv_message.key)) {
bts.Log.Gen.warn($"AsyncTCPConnection_APM._on_read key: {rcv_message.key} already handled");
return;
}
_read_key_map.Add(rcv_message.key);
if (!_is_alive)
return;
try {
SocketError read_err;
int bytes_read = _sock.EndReceive(res, out read_err);
rcv_message.advance_transfer_pointer(bytes_read);
bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read: this: {ToString()} tid: {_tid()} res: {_cb_txt(res)} bytes_read: {bytes_read} message: {res.AsyncState} _is_alive: {_is_alive} read_err: {read_err}");
if (bytes_read == 0) {
bts.Log.Gen.error($"AsyncTCPConnection_APM._on_read this: {this} read 0 bytes, throwing exception");
_handle_error(new ErrorMessage(new ApplicationException("Read 0 bytes, peer socket is likely shutdown"), "_on_read"));
return;
}
//return;
if (read_err == SocketError.Success) {
if (rcv_message.fully_transfered()) {
if (rcv_message.is_reading_size()) {
var msg_size = _on_read_size_completely(rcv_message);
if (msg_size != null && msg_size.HasValue) {
_read_data(msg_size.Value);
}
return;
} else {
// data is read
int msg_type = rcv_message.read_int32();
bts.Log.assert(msg_type == HB_MSG_SIGNAL || msg_type == DATA_MSG_SIGNAL);
if (msg_type == DATA_MSG_SIGNAL) {
_on_read_data_completely(rcv_message);
_read_next_msg();
return;
return;
} else if (msg_type == HB_MSG_SIGNAL) {
_on_read_hb_completely(rcv_message);
_read_next_msg();
return;
}
// go back to reading size for a new message
//_read_size();
}
} else {
// msg has not been fully rd, we have already advanced the transfer ptr
_begin_or_resume_reading(rcv_msg: rcv_message);
}
} else {
_handle_error(new ErrorMessage(read_err, "_on_read"));
}
} catch (Exception ex) {
_handle_error(new ErrorMessage(ex, "_on_read"));
}
}
#endregion
private int? _on_read_size_completely(AsyncReceiveMessage msg) {
bts.Log.assert(msg.is_reading_size());
bts.Log.assert(msg.fully_transfered());
bts.Log.assert(msg.original_size == 4);
var msg_sz = msg.read_int32();
bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely this: {this} msg: {msg} msg_sz: {msg_sz}");
if (msg_sz > _max_msg_size) {
string err_msg = $"Encountered malformed message, prefixed_size: {msg_sz}";
_handle_error(new ErrorMessage(new ApplicationException(err_msg), "_on_read"));
return null;
} else {
// sz is determined, now we read that many bytes
//_read_data(msg_sz);
return msg_sz;
}
}
private void _on_read_hb_completely(AsyncReceiveMessage msg) {
bts.Log.assert(!msg.is_reading_size());
bts.Log.assert(msg.fully_transfered());
bts.Log.assert(msg.original_size == 4);
bts.Log.assert(msg.read_int32(0) == HB_MSG_SIGNAL);
if (!_hb_received_yet) {
_hb_received_yet = true;
bts.Log.Gen.info($"AsyncTCPConnection_APM HB received for the first time from peer {_peer_conn_info}");
}
_last_hb_received_time = DateTime.Now;
_message_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_conn_info));
}
private void _on_read_data_completely(AsyncReceiveMessage msg) {
bts.Log.assert(!msg.is_reading_size());
bts.Log.assert(msg.fully_transfered());
var type = msg.read_int32(0);
bts.Log.assert(type == DATA_MSG_SIGNAL);
bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_data_completely this: {this} msg: {msg} type: {type}");
byte[] data = msg.read_bytes(4);
///callbacks can be made through thread pool to avoid this
///thread from being blocked, by this callback
if (!_data_received_yet) {
_data_received_yet = true;
bts.Log.Gen.info($"AsyncTCPConnection_APM data received for the first time from peer {_peer_conn_info}");
}
// could be an index here
_message_handler.onData(new RemoteMessage(data, RemoteMessageType.Data, _peer_conn_info));
}
#region IDisposable Members
public void Dispose() {
// socket is closed and disposed here
lock (this) {
if (_is_alive) {
_master_worker.removeWorker(this);
_is_alive = false;
bts.Log.Gen.info($"AsycnTCPConnection.Dispose this: {this} being disposed, last HB was received at {_last_hb_received_time}");
try {
_sock.Shutdown(SocketShutdown.Both);
} catch (Exception ex) {
String.Format("AsyncTCPConnection_APM.Dispose exception: {ex} thrown when closing socket");
if (ex is OutOfMemoryException)
throw ex;
}
_message_handler.onData(new RemoteMessage(null, RemoteMessageType.Disconnection, _peer_conn_info));
}
}
}
#endregion
private void _handle_error(ErrorMessage err_msg) {
lock (this) {
Log.Gen.info($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}");
if (err_msg.oom)
throw err_msg.exception;
Dispose();
}
}
}
}
@@ -1,2 +0,0 @@
// this implementation uses SocketAsyncEventArgs
// https://learn.microsoft.com/en-us/dotnet/api/system.net.sockets.socketasynceventargs?view=net-8.0
@@ -1,2 +0,0 @@
//This implementation uses Task-based asynchronous pattern (TAP)
//https://learn.microsoft.com/en-us/dotnet/standard/asynchronous-programming-patterns/task-based-asynchronous-pattern-tap
@@ -1,65 +0,0 @@
using System.Net.Sockets;
using System.Net;
using BCG;
using System;
using System.Collections.Generic;
namespace bts {
// Unlike AsyncTCPConnectionAcceptor_APM this class is static since the requests to connect are stand alone requests
// whereas AsyncTCPConnectionAcceptor_APM is always listening for incoming connections which lends itself to more
// statefullness and hence non static
public static class AsyncTCPConnector_APM {
internal class SocketConnectionArgs {
public EndPoint ep;
public Socket sock;
public ITCPConnectionHandler handler;
public string server_url;
internal SocketConnectionArgs(EndPoint theEp, Socket the_sock, ITCPConnectionHandler the_handler,
string theServerURL) {
ep = theEp;
sock = the_sock;
handler = the_handler;
server_url = theServerURL;
}
public override string ToString() {
return $"SocketConnectionArgs: ep: {ep} server_url: {server_url}";
}
}
public static void connect_to(BlutoSubAddress server_address, ITCPConnectionHandler handler) {
bts.Log.Gen.info($"AsyncTCPConnector_APM.connectTo server: {server_address}");
// when connection is established, this sock passed to an ITCPCOnnection and sent back
Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
EndPoint ep = new IPEndPoint(IPAddress.Parse(server_address.ip), server_address.port);
sock.BeginConnect(ep, new AsyncCallback(_on_connect),
new SocketConnectionArgs(ep, sock, handler, server_address.ToString()));
}
private static void _on_connect(IAsyncResult res) {
bts.Log.Gen.info("AsyncTCPConnector_APM._on_connect " + res.ToString() + " " + res.AsyncState.ToString());
SocketConnectionArgs sc_args = res.AsyncState as SocketConnectionArgs;
try {
sc_args.sock.EndConnect(res);
} catch (SocketException socket_ex) {
bts.Log.Gen.info($"AsyncTCPConnector_APM._on_connect EndConnect failed, args: {sc_args} ex: {socket_ex}");
if (sc_args.handler.isDisposed) {
return;
} else {
System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(1000));
bts.Log.Gen.info($"AsyncTCPConnector_APM retrying connectiont to server: {sc_args.server_url}");
sc_args.sock.BeginConnect(sc_args.ep, new AsyncCallback(_on_connect), sc_args);
return;
}
}
bts.Log.Gen.info($"AsyncTCPConnector_APM._on_connect EndConnect successful args: {sc_args}");
sc_args.handler.onConnection(new AsyncTCPConnection_APM(sc_args.sock, TimeSpan.FromSeconds(1),
ITCPConnection.ConnectionSide.Client));
}
}
}
@@ -1,82 +1,82 @@
// using System; using System;
// using System.Collections.Generic; using System.Collections.Generic;
// using System.Text; using System.Text;
// using System.Net; using System.Net;
// using System.Net.Sockets; using System.Net.Sockets;
// using System.Threading; using System.Threading;
// using System.Threading.Tasks; using System.Threading.Tasks;
// //using BCG.Utilities; //using BCG.Utilities;
// //using BCG.Utilities.General; //using BCG.Utilities.General;
// namespace BCG { namespace BCG {
// internal class SocketConnectionArgs { internal class SocketConnectionArgs {
// public EndPoint ep; public EndPoint ep;
// public Socket sock; public Socket sock;
// public ITCPConnectionHandler handler; public ITCPConnectionHandler handler;
// public string serverURL; public string serverURL;
// internal SocketConnectionArgs(EndPoint theEp, Socket theSock, ITCPConnectionHandler theHandler, internal SocketConnectionArgs(EndPoint theEp, Socket theSock, ITCPConnectionHandler theHandler,
// string theServerURL) { string theServerURL) {
// ep = theEp; ep = theEp;
// sock = theSock; sock = theSock;
// handler = theHandler; handler = theHandler;
// serverURL = theServerURL; serverURL = theServerURL;
// } }
// } }
// public class ClientToServerManagedTCPConnector { public class ClientToServerManagedTCPConnector {
// private static Random myRnd = new Random(DateTime.Now.Millisecond); private static Random myRnd = new Random(DateTime.Now.Millisecond);
// private static TimeSpan myHBTimeSpan = new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval); private static TimeSpan myHBTimeSpan = new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval);
// private static int myWaitTimeBeforeRetryingUpperLimit = 5000, private static int myWaitTimeBeforeRetryingUpperLimit = 5000,
// myWaitTimeBeforeRetryingLowerLimit = 1000; myWaitTimeBeforeRetryingLowerLimit = 1000;
// public static void connectTo(BlutoSubAddress serverAddress, ITCPConnectionHandler handler) { public static void connectTo(BlutoSubAddress serverAddress, ITCPConnectionHandler handler) {
// Console.WriteLine($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}"); Console.WriteLine($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
// Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
// EndPoint ep = new IPEndPoint(IPAddress.Parse(serverAddress.ip), serverAddress.port); EndPoint ep = new IPEndPoint(IPAddress.Parse(serverAddress.ip), serverAddress.port);
// sock.BeginConnect(ep, new AsyncCallback(myOnConnect), new SocketConnectionArgs(ep, sock, handler, serverAddress.ToString())); sock.BeginConnect(ep, new AsyncCallback(myOnConnect), new SocketConnectionArgs(ep, sock, handler, serverAddress.ToString()));
// } }
// public static async Task connect_to_async(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) { public static async Task connect_to_async(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) {
// bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}"); bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
// await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts); await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts);
// } }
// public static void connect_to(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) { public static void connect_to(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) {
// bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}"); bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
// Task.Run(async () => { Task.Run(async () => {
// await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts); await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts);
// }); });
// } }
// private static void myOnConnect(IAsyncResult res) { private static void myOnConnect(IAsyncResult res) {
// Console.WriteLine("ClientToServerManagedTCPConnector.myOnConnect " + res.ToString() + " " + res.AsyncState.ToString()); Console.WriteLine("ClientToServerManagedTCPConnector.myOnConnect " + res.ToString() + " " + res.AsyncState.ToString());
// SocketConnectionArgs args = res.AsyncState as SocketConnectionArgs; SocketConnectionArgs args = res.AsyncState as SocketConnectionArgs;
// try { try {
// args.sock.EndConnect(res); args.sock.EndConnect(res);
// } catch (SocketException sex) { } catch (SocketException sex) {
// string errMsg = String.Format("{0}: Connection to {1} failed.", DateTime.Now.ToString(), args.serverURL.ToString()); string errMsg = String.Format("{0}: Connection to {1} failed.", DateTime.Now.ToString(), args.serverURL.ToString());
// //BlueEventLog.logEvent(sex.Message + ";" + errMsg, PS3LogLevels.disconnection); //BlueEventLog.logEvent(sex.Message + ";" + errMsg, PS3LogLevels.disconnection);
// //Console.WriteLine(errMsg); //Console.WriteLine(errMsg);
// if (args.handler.isDisposed) { if (args.handler.isDisposed) {
// string doneMsg = String.Format("{0}: Connection to {1} was disposed. Not Retrying.", DateTime.Now.ToString(), args.serverURL.ToString()); string doneMsg = String.Format("{0}: Connection to {1} was disposed. Not Retrying.", DateTime.Now.ToString(), args.serverURL.ToString());
// //Console.WriteLine(doneMsg); //Console.WriteLine(doneMsg);
// // BlueEventLog.logEvent(doneMsg, PS3LogLevels.disconnection); // BlueEventLog.logEvent(doneMsg, PS3LogLevels.disconnection);
// return; return;
// } else { } else {
// string retryMsg = String.Format("{0}: Retrying Connection to {1}.", DateTime.Now.ToString(), args.serverURL.ToString()); string retryMsg = String.Format("{0}: Retrying Connection to {1}.", DateTime.Now.ToString(), args.serverURL.ToString());
// //Console.WriteLine(retryMsg); //Console.WriteLine(retryMsg);
// //BlueEventLog.logEvent(retryMsg, PS3LogLevels.disconnection); //BlueEventLog.logEvent(retryMsg, PS3LogLevels.disconnection);
// int waitTime = myRnd.Next(myWaitTimeBeforeRetryingLowerLimit, myWaitTimeBeforeRetryingUpperLimit); int waitTime = myRnd.Next(myWaitTimeBeforeRetryingLowerLimit, myWaitTimeBeforeRetryingUpperLimit);
// Thread.Sleep(waitTime); Thread.Sleep(waitTime);
// args.sock.BeginConnect(args.ep, new AsyncCallback(myOnConnect), args); args.sock.BeginConnect(args.ep, new AsyncCallback(myOnConnect), args);
// return; return;
// } }
// } }
// //BlueEventLog.logEvent(String.Format("Successful connection made to {0}.", args.serverURL.ToString()), PS3LogLevels.connection); //BlueEventLog.logEvent(String.Format("Successful connection made to {0}.", args.serverURL.ToString()), PS3LogLevels.connection);
// args.handler.onConnection(new bts.AsyncTCPConnection_APM(args.sock, myHBTimeSpan, ITCPConnection.ConnectionSide.Client)); args.handler.onConnection(new AsyncTCPConnection(args.sock, myHBTimeSpan));
// } }
// } }
// } }
@@ -9,10 +9,6 @@ namespace BCG {
/// on top of the sockets (used on both server and client) /// on top of the sockets (used on both server and client)
/// </summary> /// </summary>
public interface ITCPConnection : IDisposable { public interface ITCPConnection : IDisposable {
public enum ConnectionSide {
Client,
Server
}
/// <summary> /// <summary>
/// The delegate is fired signalling if the data send was /// The delegate is fired signalling if the data send was
/// successful or not /// successful or not
@@ -22,20 +22,20 @@ namespace BCG {
return portNo; return portNo;
} }
// public static int activate(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) { public static int activate(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) {
// lock (mySyncObject) { lock (mySyncObject) {
// if (myPortToSingleServerAcceptorMap.ContainsKey(port)) { if (myPortToSingleServerAcceptorMap.ContainsKey(port)) {
// string msg = string.Format("Error: Application is already accepting clients on port {0}", port); string msg = string.Format("Error: Application is already accepting clients on port {0}", port);
// // BlueEventLog.logEvent(msg, BlueEventLog2Level.Error); // BlueEventLog.logEvent(msg, BlueEventLog2Level.Error);
// return port; return port;
// } else { } else {
// SingleServerToClientManagedTCPConnectionAcceptor theAcceptor = new SingleServerToClientManagedTCPConnectionAcceptor(); SingleServerToClientManagedTCPConnectionAcceptor theAcceptor = new SingleServerToClientManagedTCPConnectionAcceptor();
// int assignedport = theAcceptor.activate(handler, port, clientHBTimeSpan); int assignedport = theAcceptor.activate(handler, port, clientHBTimeSpan);
// myPortToSingleServerAcceptorMap[assignedport] = theAcceptor; myPortToSingleServerAcceptorMap[assignedport] = theAcceptor;
// return assignedport; return assignedport;
// } }
// } //end lock } //end lock
// } //end activate code } //end activate code
public static int accept_connections(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) { public static int accept_connections(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) {
lock (mySyncObject) { lock (mySyncObject) {
@@ -89,7 +89,7 @@ namespace BCG {
} }
newSock.LingerState = new LingerOption(false, 0); newSock.LingerState = new LingerOption(false, 0);
newSock.Blocking = false; newSock.Blocking = false;
ITCPConnection newConn = new bts.AsyncTCPConnection_APM(newSock, myHBTimeSpan, ITCPConnection.ConnectionSide.Server); ITCPConnection newConn = new AsyncTCPConnection(newSock, myHBTimeSpan);
myHandler.onConnection(newConn); myHandler.onConnection(newConn);
myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { }); myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { });
string ip; int port; string ip; int port;
@@ -11,34 +11,38 @@ using System.Xml.Serialization;
using System.Reflection; using System.Reflection;
using BCG.Utilities.General; using BCG.Utilities.General;
using System.Net.WebSockets; using System.Net.WebSockets;
using bts;
using System.Threading;
using Google.Protobuf.WellKnownTypes;
namespace BCG { namespace BCG
{
// utility functions for TCP socket async communications. Does the following // utility functions for TCP socket async communications. Does the following
// client to server connect // client to server connect
// server to client connect accepting // server to client connect accepting
// send data // send data
// receive data // receive data
public static class TCPSocketUtils { public static class TCPSocketUtils
{
#region socket opening and connecting #region socket opening and connecting
// keeps accepting connections forever // keeps accepting connections forever
public static async Task accept_connections_async(string ip, int port, ITCPConnectionHandler handler, public static async Task accept_connections_async(string ip, int port, ITCPConnectionHandler handler,
TimeSpan wait_after_error) { TimeSpan wait_after_error)
{
IPAddress ipAddress = String.IsNullOrEmpty(ip) ? IPAddress.Parse(ip) : IPAddress.Any; IPAddress ipAddress = String.IsNullOrEmpty(ip) ? IPAddress.Parse(ip) : IPAddress.Any;
var ip_end_point = new IPEndPoint(ipAddress, port); var ip_end_point = new IPEndPoint(ipAddress, port);
Socket listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); Socket listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
listener.Bind(ip_end_point); listener.Bind(ip_end_point);
listener.Listen(10); listener.Listen(10);
Socket client_socket = null; Socket client_socket = null;
while (true) { while (true)
try { {
try
{
// keeps on accepting connections // keeps on accepting connections
client_socket = await listener.AcceptAsync(); client_socket = await listener.AcceptAsync();
ITCPConnection new_conn = new bts.AsyncTCPConnection_APM(client_socket, TimeSpan.FromSeconds(1), ITCPConnection.ConnectionSide.Server); AsyncTCPConnection2 new_conn = new(client_socket, AsyncTCPConnection2.ConnectionSide.Server);
handler.onConnection(new_conn); handler.onConnection(new_conn);
} catch (Exception ex) { }
catch (Exception ex)
{
bts.Log.Gen.error($"TCPConnection.accept ex: {ex} trying again"); bts.Log.Gen.error($"TCPConnection.accept ex: {ex} trying again");
Task.Delay(wait_after_error); Task.Delay(wait_after_error);
} }
@@ -48,20 +52,25 @@ namespace BCG {
// connects to the server and ona successful connection, calls con__handler.on_connection_async with an ITCPConnection. // connects to the server and ona successful connection, calls con__handler.on_connection_async with an ITCPConnection.
// In case of failures to connect it waits for a given amount of time before retrying // In case of failures to connect it waits for a given amount of time before retrying
public static async Task<Socket> connect_to_server_async(string ip, int server_port, ITCPConnectionHandler conn_handler, public static async Task<Socket> connect_to_server_async(string ip, int server_port, ITCPConnectionHandler conn_handler,
TimeSpan wait_after_failed_attempt, int n_max_attempts = 1024) { TimeSpan wait_after_failed_attempt, int n_max_attempts = 1024)
{
Socket client_socket = null; Socket client_socket = null;
for (int i = 0; i < n_max_attempts; i++) { for (int i = 0; i < n_max_attempts; i++)
try { {
try
{
string log_conn = $"{ip}:{server_port}"; string log_conn = $"{ip}:{server_port}";
bts.Log.Gen.info($"AsyncTCPConnection2.connect_to_server conn: {log_conn} attempt no: {i}"); bts.Log.Gen.info($"AsyncTCPConnection2.connect_to_server conn: {log_conn} attempt no: {i}");
IPAddress server_ip_address = IPAddress.Parse(ip); IPAddress server_ip_address = IPAddress.Parse(ip);
client_socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); client_socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
await client_socket.ConnectAsync(server_ip_address, server_port); await client_socket.ConnectAsync(server_ip_address, server_port);
bts.Log.Gen.info($"AsyncTCPConnection2.connect_to_server conn: {log_conn} successfully connected after {i} attempts"); bts.Log.Gen.info($"AsyncTCPConnection2.connect_to_server conn: {log_conn} successfully connected after {i} attempts");
ITCPConnection new_conn = new bts.AsyncTCPConnection_APM(client_socket, TimeSpan.FromSeconds(1), ITCPConnection.ConnectionSide.Client); ITCPConnection new_conn = new AsyncTCPConnection2(client_socket, AsyncTCPConnection2.ConnectionSide.Client);
conn_handler.onConnection(new_conn); conn_handler.onConnection(new_conn);
break; break;
} catch (Exception ex) { }
catch (Exception ex)
{
bts.Log.Gen.error($"AsyncTCPConnection2.connect_to_server threw ex: {ex}, will reattempt now"); bts.Log.Gen.error($"AsyncTCPConnection2.connect_to_server threw ex: {ex}, will reattempt now");
Task.Delay(wait_after_failed_attempt); Task.Delay(wait_after_failed_attempt);
} }
@@ -74,116 +83,31 @@ namespace BCG {
// prefixes the size of the data and sends it as 2 sends, for size and data // prefixes the size of the data and sends it as 2 sends, for size and data
// (can be done as one send perhaps) // (can be done as one send perhaps)
public static async Task send_bytes_async(NetworkStream nw_stream, byte[] data_buff) { public static async Task send_bytes_async(NetworkStream nw_stream, byte[] data_buff)
Log.Gen.info($"send_bytes_async data_buff: {data_buff.Length}"); {
// var data_sz = data_buff.Length; var data_sz = data_buff.Length;
byte[] sz_buff = new byte[4]; byte[] sz_buff = new byte[4];
// // send the size first // aend the size first
BinaryWriter bw = new BinaryWriter(new MemoryStream(sz_buff)); BinaryWriter bw = new BinaryWriter(new MemoryStream(sz_buff));
bw.Write(data_buff.Length); bw.Write(data_sz);
await SendMessageAsync(nw_stream.Socket, new Memory<byte>(sz_buff)); await nw_stream.WriteAsync(sz_buff, 0, 4);
// await nw_stream.WriteAsync(sz_buff, 0, 4); await nw_stream.WriteAsync(data_buff, 0, data_buff.Length);
// await nw_stream.WriteAsync(data_buff, 0, data_buff.Length);
Memory<byte> mem = new Memory<byte>(data_buff);
await SendMessageAsync(nw_stream.Socket, mem);
} }
public static async Task SendMessageAsync(Socket sock, Memory<byte> mem_buff) {
// use ManualResetEvent to make it non async
Log.Gen.info($"SendMessageAsync mem: {mem_buff.Length}");
SocketAsyncEventArgs evargs = new SocketAsyncEventArgs();
evargs.SetBuffer(mem_buff);
var tcs = new TaskCompletionSource<bool>();
evargs.Completed += (s, e) => {
Log.Gen.info($"SendMessageAsync completed asynchronously e: {sock_event_args_txt(e)}");
tcs.SetResult(true);
};
bool completed = !sock.SendAsync(evargs);
if (completed) {
Log.Gen.info($"SendMessageAsync completed synchronously e: {sock_event_args_txt(evargs)}");
tcs.SetResult(true);
}
await tcs.Task;
}
// reads the bytes on a connected socket, by doing the opposite of the above, // reads the bytes on a connected socket, by doing the opposite of the above,
// reads the size first and then the data // reads the size first and then the data
public static async Task<byte[]> read_bytes_async(NetworkStream nw_stream) { public static async Task<byte[]> read_bytes_async(NetworkStream nw_stream)
// byte[] data_buffer; {
// try { byte[] sz_buffer = new byte[4];
// byte[] sz_buffer = new byte[4]; //Asynchronously reads a sequence of bytes from the current stream and advances the position within the stream by the number of bytes read.
// //Asynchronously reads a sequence of bytes from the current stream and advances the position within the stream by the number of bytes read. int bytesRead = await nw_stream.ReadAsync(sz_buffer, 0, sz_buffer.Length);
// int bytesRead = await nw_stream.ReadAsync(sz_buffer, 0, sz_buffer.Length); BinaryReader br = new BinaryReader(new MemoryStream(sz_buffer));
// BinaryReader br = new BinaryReader(new MemoryStream(sz_buffer)); int msg_sz = br.ReadInt32();
// int msg_sz = br.ReadInt32(); byte[] data_buffer = new byte[msg_sz];
// data_buffer = new byte[msg_sz]; bytesRead = await nw_stream.ReadAsync(data_buffer, 0, data_buffer.Length);
// bytesRead = await nw_stream.ReadAsync(data_buffer, 0, data_buffer.Length); //Console.WriteLine($"read_bytes sz: {data_buffer.Length}");
// Log.Gen.info($"read_bytes_async intended: {msg_sz} received: {bytesRead}"); return data_buffer;
// } catch (Exception ex) {
// Log.Gen.error($"read_bytes_async exception: {ex}");
// return new byte[] { };
// }
// //Console.WriteLine($"read_bytes sz: {data_buffer.Length}");
// return data_buffer;
return ReceiveSizePrefixedMessageAsync(nw_stream.Socket);
}
static string sock_event_args_txt(SocketAsyncEventArgs seargs) {
return $"err: {seargs.SocketError} bytes: {seargs.BytesTransferred}";
}
// this is a limited sync wrapper ovr the async ReceiveAsync
static byte[] ReceiveSizePrefixedMessageAsync(Socket socket) {
const int buffer_size = 4; // Size of the buffer to read the message size
var size_buffer = new byte[buffer_size];
var size_event_args = new SocketAsyncEventArgs();
size_event_args.SetBuffer(size_buffer, 0, buffer_size);
var size_completed_event = new ManualResetEvent(false);
size_event_args.Completed += (sender, e) => {
Log.Gen.info($"ReceiveSizePrefixedMessageAsync async completed (msg len) e: {sock_event_args_txt(e)}");
size_completed_event.Set(); // Signal that the size receive operation is complete
};
if (!socket.ReceiveAsync(size_event_args)) {
Log.Gen.info($"ReceiveSizePrefixedMessageAsync msg size read completed synchronously e: {sock_event_args_txt(seargs: size_event_args)}");
// If ReceiveAsync completed synchronously, set the event manually
size_completed_event.Set();
}
size_completed_event.WaitOne(); // Wait for the size receive operation to complete
int message_size = BitConverter.ToInt32(size_buffer, 0);
//Log.assert_d(message_size > 0, $"message_size: {message_size}");
if (message_size <= 0)
throw new ApplicationException($"msg_size: {message_size} not allowed");
Log.Gen.info($"ReceiveSizePrefixedMessageAsync message_size: {message_size}");
// at this ppint message sz has been read, now read the data
var message_buffer = new byte[message_size];
var message_event_args = new SocketAsyncEventArgs();
message_event_args.SetBuffer(message_buffer, 0, message_size);
var message_completed_event = new ManualResetEvent(false);
message_event_args.Completed += (sender, e) => {
Log.Gen.info($"ReceiveSizePrefixedMessageAsync async completed (msg) e: {sock_event_args_txt(e)}");
message_completed_event.Set(); // Signal that the message receive operation is complete
};
if (!socket.ReceiveAsync(message_event_args)) {
Log.Gen.info($"ReceiveSizePrefixedMessageAsync msg read completed synchronously e: {sock_event_args_txt(seargs: message_event_args)}");
// If ReceiveAsync completed synchronously, set the event manually
message_completed_event.Set();
}
message_completed_event.WaitOne(); // Wait for the message receive operation to complete
return message_buffer;
} }
#endregion
} }
#endregion
} }
+5 -8
View File
@@ -4,22 +4,19 @@ using System.IO;
namespace BCG { namespace BCG {
/// <summary> /// <summary>
/// this is the data returned from the pubsub server /// this is the data returned from the
/// pubsub server
/// </summary> /// </summary>
public class PubSubUpdate { public class PubSubUpdate {
public PubSubUpdate(string tag, string the_tag_type, Stream body) { public PubSubUpdate(string tag, string tagType, Stream body) { this.tag = tag; this.body = body; this.tagType = tagType; }
this.tag = tag;
this.body = body;
this.tag_type = the_tag_type;
}
public string tag; public string tag;
public Stream body; public Stream body;
public string tag_type; public string tagType;
public override string ToString() { public override string ToString() {
return $"PubSubUpdate tag: {tag} tag_type: {tag_type} len: {body.Position}"; return $"PubSubUpdate tag: {tag} tag_type: {tagType} len: {body.Position}";
} }
}; };
} }
@@ -5,16 +5,18 @@ using System.IO;
using System.Collections.Generic; using System.Collections.Generic;
using mbts; using mbts;
namespace BCG { namespace BCG
public delegate void PubSubServerConnectionStatusChange(string peerURL, bool connection); {
public delegate void PubSubServerDataDelegate(string sender, byte[] data); public delegate void ConnectionStatusChange(string peerURL, bool connection);
public delegate void DataDelegate(string sender, byte[] data);
/// <summary> /// <summary>
/// Encapsulates PubsubServer's communication with the /// Encapsulates PubsubServer's communication with the
/// rest of the world. Has send and get methods, and also /// rest of the world. Has send and get methods, and also
/// an event to tell if a client connects or disconnects. /// an event to tell if a client connects or disconnects.
/// this event is prompted by the doWork. /// this event is prompted by the doWork.
/// </summary> /// </summary>
public interface IPubsubMessageHub : IWorker { public interface IPubsubMessageHub : IWorker
{
/// <summary> /// <summary>
/// The port that the message hub is listening on. Return -1 if you're not listening on a port, i.e. for hte case of multicast /// The port that the message hub is listening on. Return -1 if you're not listening on a port, i.e. for hte case of multicast
@@ -27,14 +29,14 @@ namespace BCG {
/// getUpdates. The delegate will be fired on an arbitrary thread. /// getUpdates. The delegate will be fired on an arbitrary thread.
/// </summary> /// </summary>
/// <param name="?"></param> /// <param name="?"></param>
void registerDataCallback(PubSubServerDataDelegate del); void registerDataCallback(DataDelegate del);
/// <summary> /// <summary>
/// registers a connection callback. this callback /// registers a connection callback. this callback
/// will be fired within the doWork call. /// will be fired within the doWork call.
/// </summary> /// </summary>
/// <param name="del"></param> /// <param name="del"></param>
/// <param name="register"></param> /// <param name="register"></param>
void registerConnectionCallback(PubSubServerConnectionStatusChange del); void registerConnectionCallback(ConnectionStatusChange del);
/// <summary> /// <summary>
/// this is where this interface starts communication with the /// this is where this interface starts communication with the
/// rest of the world. Obvioulsy, the resgiters should be /// rest of the world. Obvioulsy, the resgiters should be
@@ -22,8 +22,8 @@ namespace BCG {
} }
private bool myActivated = false; private bool myActivated = false;
private PubSubServerConnectionStatusChange myConnCb; private ConnectionStatusChange myConnCb;
private PubSubServerDataDelegate myDataCb; private DataDelegate myDataCb;
/// <summary> /// <summary>
/// client to connection /// client to connection
@@ -77,12 +77,12 @@ namespace BCG {
#region IPubsubMessageHub Members #region IPubsubMessageHub Members
public void registerDataCallback(PubSubServerDataDelegate del) { public void registerDataCallback(DataDelegate del) {
// BlueEventLog.logAssert(!myActivated, "connection callbacks should be registered before the call to activate"); // BlueEventLog.logAssert(!myActivated, "connection callbacks should be registered before the call to activate");
myDataCb = del; myDataCb = del;
} }
public void registerConnectionCallback(PubSubServerConnectionStatusChange del) { public void registerConnectionCallback(ConnectionStatusChange del) {
//BlueEventLog.logAssert(!myActivated, "data callbacks should be registered before the call to activate"); //BlueEventLog.logAssert(!myActivated, "data callbacks should be registered before the call to activate");
myConnCb = del; myConnCb = del;
} }
@@ -4,22 +4,25 @@ using System.Threading;
using System.IO; using System.IO;
using mbts; using mbts;
namespace BCG { namespace BCG
{
/// <summary> /// <summary>
/// Wraps around all the IPubsubMessageHub's that pubsub supports. /// Wraps around all the IPubsubMessageHub's that pubsub supports.
/// ie MSMQ and TCP to begin with. /// ie MSMQ and TCP to begin with.
/// </summary> /// </summary>
public class ServerMessageHub : IPubsubMessageHub { public class ServerMessageHub : IPubsubMessageHub
{
private static ServerMessageHub mySelf = null; private static ServerMessageHub mySelf = null;
private static object myInstanceLock = new object(); private static object myInstanceLock = new object();
private Dictionary<string, IPubsubMessageHub> myHubs = new Dictionary<string, IPubsubMessageHub>(); private Dictionary<string, IPubsubMessageHub> myHubs = new Dictionary<string, IPubsubMessageHub>();
private SyncQueue<Pair<string, byte[]>> myReceivingQueue = new SyncQueue<Pair<string, byte[]>>(); private SyncQueue<Pair<string, byte[]>> myReceivingQueue = new SyncQueue<Pair<string, byte[]>>();
private PubSubServerConnectionStatusChange myConnectionCb; private ConnectionStatusChange myConnectionCb;
private PubSubServerDataDelegate myDataCb; private DataDelegate myDataCb;
private MasterWorker myMasterWorker = MasterWorker.getInstance(false); private MasterWorker myMasterWorker = MasterWorker.getInstance(false);
protected ServerMessageHub() : this(PubSub3Config.instance.PS3_ServerPort) { } protected ServerMessageHub() : this(PubSub3Config.instance.PS3_ServerPort) { }
protected ServerMessageHub(int portNumber) { protected ServerMessageHub(int portNumber)
{
// TODO: no more MSMQ // TODO: no more MSMQ
// if (!PubSub3Config.instance.PS3_DisableMSMQ) // if (!PubSub3Config.instance.PS3_DisableMSMQ)
// myHubs.Add(MSMQSubAddress.prefix, new MSMQMessageHub()); // myHubs.Add(MSMQSubAddress.prefix, new MSMQMessageHub());
@@ -32,14 +35,19 @@ namespace BCG {
as IPubsubMessageHub); as IPubsubMessageHub);
} }
public int port { public int port
{
get { return myHubs[BlutoSubAddress.prefix].port; } get { return myHubs[BlutoSubAddress.prefix].port; }
} }
public static ServerMessageHub instance { public static ServerMessageHub instance
get { {
if (mySelf == null) { get
lock (myInstanceLock) { {
if (mySelf == null)
{
lock (myInstanceLock)
{
if (mySelf == null) if (mySelf == null)
mySelf = new ServerMessageHub(); mySelf = new ServerMessageHub();
} }
@@ -52,39 +60,47 @@ namespace BCG {
/* Builds the server message hub with a specified port number. Only works if the server message hub has not already been created. /* Builds the server message hub with a specified port number. Only works if the server message hub has not already been created.
* */ * */
public static ServerMessageHub getInstance(int portNumber) { public static ServerMessageHub getInstance(int portNumber)
if (mySelf == null) { {
lock (myInstanceLock) { if (mySelf == null)
{
lock (myInstanceLock)
{
if (mySelf == null) if (mySelf == null)
mySelf = new ServerMessageHub(portNumber); mySelf = new ServerMessageHub(portNumber);
} }
} else }
else
throw new ArgumentException("Error: Cannot specify a port number for the ServerMessageHub instance if it has already been created."); throw new ArgumentException("Error: Cannot specify a port number for the ServerMessageHub instance if it has already been created.");
return mySelf; return mySelf;
} }
#region IPubsubMessageHub Members #region IPubsubMessageHub Members
public void registerDataCallback(PubSubServerDataDelegate del) { myDataCb = del; } public void registerDataCallback(DataDelegate del) { myDataCb = del; }
public void registerConnectionCallback(PubSubServerConnectionStatusChange del) { myConnectionCb = del; } public void registerConnectionCallback(ConnectionStatusChange del) { myConnectionCb = del; }
public void activate() { public void activate()
{
string msg = "PubSubServer started .. initializing TCP transport components .. "; string msg = "PubSubServer started .. initializing TCP transport components .. ";
Console.WriteLine(msg); Console.WriteLine(msg);
foreach (IPubsubMessageHub hub in myHubs.Values) { foreach (IPubsubMessageHub hub in myHubs.Values)
hub.registerDataCallback(new PubSubServerDataDelegate(myOnData)); {
hub.registerConnectionCallback(new PubSubServerConnectionStatusChange(myOnConnection)); hub.registerDataCallback(new DataDelegate(myOnData));
hub.registerConnectionCallback(new ConnectionStatusChange(myOnConnection));
hub.activate(); hub.activate();
} }
myMasterWorker.addWorker(this, new TimeSpan(0, 0, 0, 0, 250)); myMasterWorker.addWorker(this, new TimeSpan(0, 0, 0, 0, 250));
} }
public List<Pair<string, byte[]>> getUpdates(TimeSpan timeout, int maxElements) { public List<Pair<string, byte[]>> getUpdates(TimeSpan timeout, int maxElements)
{
return myReceivingQueue.dequeueMultipleElements(timeout, maxElements); return myReceivingQueue.dequeueMultipleElements(timeout, maxElements);
} }
public bool send(string receiver, byte[] data) { public bool send(string receiver, byte[] data)
{
if (receiver.StartsWith(BlutoSubAddress.prefix)) if (receiver.StartsWith(BlutoSubAddress.prefix))
return myHubs[BlutoSubAddress.prefix].send(receiver, data); return myHubs[BlutoSubAddress.prefix].send(receiver, data);
else if (receiver.StartsWith(MSMQSubAddress.prefix)) else if (receiver.StartsWith(MSMQSubAddress.prefix))
@@ -93,7 +109,8 @@ namespace BCG {
return false; return false;
} }
public bool send(string receiver, Stream str) { public bool send(string receiver, Stream str)
{
if (receiver.StartsWith(BlutoSubAddress.prefix)) if (receiver.StartsWith(BlutoSubAddress.prefix))
return myHubs[BlutoSubAddress.prefix].send(receiver, str); return myHubs[BlutoSubAddress.prefix].send(receiver, str);
else if (receiver.StartsWith(MSMQSubAddress.prefix)) else if (receiver.StartsWith(MSMQSubAddress.prefix))
@@ -109,7 +126,8 @@ namespace BCG {
/// <summary> /// <summary>
/// connection callback will be called within this doWork /// connection callback will be called within this doWork
/// </summary> /// </summary>
public void doWork() { public void doWork()
{
foreach (IPubsubMessageHub hub in myHubs.Values) foreach (IPubsubMessageHub hub in myHubs.Values)
hub.doWork(); hub.doWork();
} }
@@ -120,18 +138,21 @@ namespace BCG {
public static string internalPublisherConnection = "InternalPublisher"; public static string internalPublisherConnection = "InternalPublisher";
public void internalSend(byte[] data) { public void internalSend(byte[] data)
{
myReceivingQueue.enqueue(new Pair<string, byte[]>(internalPublisherConnection, data)); myReceivingQueue.enqueue(new Pair<string, byte[]>(internalPublisherConnection, data));
} }
private void myOnData(string sender, byte[] msg) { private void myOnData(string sender, byte[] msg)
{
if (myDataCb != null) if (myDataCb != null)
myOnData(sender, msg); myOnData(sender, msg);
else else
myReceivingQueue.enqueue(new Pair<string, byte[]>(sender, msg)); myReceivingQueue.enqueue(new Pair<string, byte[]>(sender, msg));
} }
private void myOnConnection(string clientURL, bool connected) { private void myOnConnection(string clientURL, bool connected)
{
if (myConnectionCb != null) if (myConnectionCb != null)
myConnectionCb(clientURL, connected); myConnectionCb(clientURL, connected);
} }
@@ -58,7 +58,7 @@ namespace BCG {
myReceivingProcThread = new Thread(new ThreadStart(myReceivingThreadProc)); myReceivingProcThread = new Thread(new ThreadStart(myReceivingThreadProc));
myMessageHub = ServerMessageHub.instance; myMessageHub = ServerMessageHub.instance;
myMessageHub.registerConnectionCallback(new PubSubServerConnectionStatusChange(myHandleConnections)); myMessageHub.registerConnectionCallback(new ConnectionStatusChange(myHandleConnections));
} }
@@ -55,7 +55,7 @@ namespace BCG {
myTagToPubsubMap[tag].onSub(sub); myTagToPubsubMap[tag].onSub(sub);
if (onSingleTagChange != null) if (onSingleTagChange != null)
onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribing_app)); onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribingApp));
} }
if (onTagMapChanged != null) if (onTagMapChanged != null)
onTagMapChanged(sub.tags); onTagMapChanged(sub.tags);
@@ -68,10 +68,10 @@ namespace BCG {
///and publishers for the tag then remove it. ///and publishers for the tag then remove it.
myTagToPubsubMap[tag].onUnsub(unsub); myTagToPubsubMap[tag].onUnsub(unsub);
if (onSingleTagChange != null) if (onSingleTagChange != null)
onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribing_app)); onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribingApp));
} else { } else {
string warningMsg = String.Format("{0} was never published or subscribed on this pubsub, hence" + string warningMsg = String.Format("{0} was never published or subscribed on this pubsub, hence" +
" the unsub call by app {1} is voided", tag, unsub.unsubscribing_app); " the unsub call by app {1} is voided", tag, unsub.unsubscribingApp);
//BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning); //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
} }
} }
@@ -1,19 +1,19 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Text; using System.Text;
using BCG.Utilities; using BCG.Utilities;
using mbts; using mbts;
namespace BCG { namespace BCG
{
/// <summary> /// <summary>
/// SingleTagPubSubMap consists of all the state w.r.t to /// SingleTagPubSubMap consists of all the state w.r.t to
/// publishing and subscription of a single tag. this information /// publishing and subscription of a single tag. this information
/// consists of the publisher, and a map of the subscriber /// consists of the publisher, and a map of the subscriber
/// connection vs the apps they're coming from /// connection vs the apps they're coming from
/// </summary> /// </summary>
internal class SingleTagPubSubMap { internal class SingleTagPubSubMap
{
private string myTag; private string myTag;
/// <summary> /// <summary>
/// map of clientURL to client app (machine;app;pid;clientURL) /// map of clientURL to client app (machine;app;pid;clientURL)
@@ -29,7 +29,8 @@ namespace BCG {
/// </summary> /// </summary>
List<Pair<string, string>> myDeadPubConnToApp = new List<Pair<string, string>>(); List<Pair<string, string>> myDeadPubConnToApp = new List<Pair<string, string>>();
public SingleTagPubSubMap(string tag) { public SingleTagPubSubMap(string tag)
{
myTag = tag; myTag = tag;
} }
@@ -37,7 +38,8 @@ namespace BCG {
/// Returns a list all the active connections that /// Returns a list all the active connections that
/// have subscribed to this tag /// have subscribed to this tag
/// </summary> /// </summary>
public ICollection<string> subscriberConnections { public ICollection<string> subscriberConnections
{
get { return mySubConnToApp.Keys; } get { return mySubConnToApp.Keys; }
} }
@@ -45,7 +47,8 @@ namespace BCG {
/// Returns a list of all the applications that have /// Returns a list of all the applications that have
/// subscribed to this tag /// subscribed to this tag
/// </summary> /// </summary>
public ICollection<string> subscripingApps { public ICollection<string> subscripingApps
{
get { return mySubConnToApp.Values; } get { return mySubConnToApp.Values; }
} }
@@ -54,7 +57,8 @@ namespace BCG {
/// connections as key and the application /// connections as key and the application
/// as value /// as value
/// </summary> /// </summary>
public Dictionary<string, string> subscribers { public Dictionary<string, string> subscribers
{
get { return mySubConnToApp; } get { return mySubConnToApp; }
} }
@@ -62,7 +66,8 @@ namespace BCG {
/// Returns a dictionary for all active publishing /// Returns a dictionary for all active publishing
/// connection as key and the application as value /// connection as key and the application as value
/// </summary> /// </summary>
public Dictionary<string, string> activePublishers { public Dictionary<string, string> activePublishers
{
get { return myActivePubConnToApp; } get { return myActivePubConnToApp; }
} }
@@ -70,7 +75,8 @@ namespace BCG {
/// Returns a list of pair of dead connections and /// Returns a list of pair of dead connections and
/// theur corresponding applications /// theur corresponding applications
/// </summary> /// </summary>
public ICollection<Pair<string, string>> deadPublishers { public ICollection<Pair<string, string>> deadPublishers
{
get { return myDeadPubConnToApp; } get { return myDeadPubConnToApp; }
} }
@@ -81,14 +87,18 @@ namespace BCG {
/// </summary> /// </summary>
/// <param name="sub"></param> /// <param name="sub"></param>
/// <returns></returns> /// <returns></returns>
public bool onSub(SubMetaData sub) { public bool onSub(SubMetaData sub)
if (mySubConnToApp.ContainsKey(sub.connection)) { {
if (mySubConnToApp.ContainsKey(sub.connection))
{
string warningMsg = String.Format("tag {0} has already been subscribed by app {0}, subscriber {1}" string warningMsg = String.Format("tag {0} has already been subscribed by app {0}, subscriber {1}"
+ "it should unsubscribe before resubscribing", myTag, sub.connection, sub.subscribing_app); + "it should unsubscribe before resubscribing", myTag, sub.connection, sub.subscribingApp);
//BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning); //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
return false; return false;
} else { }
mySubConnToApp.Add(sub.connection, sub.subscribing_app); else
{
mySubConnToApp.Add(sub.connection, sub.subscribingApp);
return true; return true;
} }
} }
@@ -100,13 +110,17 @@ namespace BCG {
/// </summary> /// </summary>
/// <param name="unsub"></param> /// <param name="unsub"></param>
/// <returns></returns> /// <returns></returns>
public bool onUnsub(UnSubMetaData unsub) { public bool onUnsub(UnSubMetaData unsub)
if (mySubConnToApp.ContainsKey(unsub.connection)) { {
if (mySubConnToApp.ContainsKey(unsub.connection))
{
mySubConnToApp.Remove(unsub.connection); mySubConnToApp.Remove(unsub.connection);
return true; return true;
} else { }
else
{
string warningMsg = String.Format("tag {0} has already was either unsubscribed or never subscribed by app {0}, subscriber {1}", string warningMsg = String.Format("tag {0} has already was either unsubscribed or never subscribed by app {0}, subscriber {1}",
myTag, unsub.connection, unsub.unsubscribing_app); myTag, unsub.connection, unsub.unsubscribingApp);
//BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning); //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
return false; return false;
} }
@@ -119,20 +133,25 @@ namespace BCG {
/// </summary> /// </summary>
/// <param name="subscriber"></param> /// <param name="subscriber"></param>
/// <returns></returns> /// <returns></returns>
public bool onConnectionRemoveIfExists(string connection, out bool wasPublisher, out bool wasSubscriber) { public bool onConnectionRemoveIfExists(string connection, out bool wasPublisher, out bool wasSubscriber)
{
bool rv = false; bool rv = false;
if (mySubConnToApp.ContainsKey(connection)) { if (mySubConnToApp.ContainsKey(connection))
{
mySubConnToApp.Remove(connection); mySubConnToApp.Remove(connection);
wasSubscriber = true; wasSubscriber = true;
rv = true; rv = true;
} else }
else
wasSubscriber = false; wasSubscriber = false;
if (myActivePubConnToApp.ContainsKey(connection)) { if (myActivePubConnToApp.ContainsKey(connection))
{
myDeadPubConnToApp.Add(new Pair<string, string>(connection, myActivePubConnToApp[connection])); myDeadPubConnToApp.Add(new Pair<string, string>(connection, myActivePubConnToApp[connection]));
myActivePubConnToApp.Remove(connection); myActivePubConnToApp.Remove(connection);
rv = true; rv = true;
wasPublisher = true; wasPublisher = true;
} else }
else
wasPublisher = false; wasPublisher = false;
return rv; return rv;
} }
@@ -143,10 +162,14 @@ namespace BCG {
/// </summary> /// </summary>
/// <param name="pub"></param> /// <param name="pub"></param>
/// <param name="publisherChanged"></param> /// <param name="publisherChanged"></param>
public void onPub(PubMetaData pub, out bool newPublisher) { public void onPub(PubMetaData pub, out bool newPublisher)
if (myActivePubConnToApp.ContainsKey(pub.connection)) { {
if (myActivePubConnToApp.ContainsKey(pub.connection))
{
newPublisher = false; newPublisher = false;
} else { }
else
{
myActivePubConnToApp.Add(pub.connection, pub.publishingApp); myActivePubConnToApp.Add(pub.connection, pub.publishingApp);
newPublisher = true; newPublisher = true;
} }
@@ -6,79 +6,92 @@ using BCG.Utilities;
using BCG.Utilities.General; using BCG.Utilities.General;
using mbts; using mbts;
using mbts.spark; using mbts.spark;
using System.Linq;
namespace BCG { namespace BCG {
/// <summary> /// <summary>
/// Deals with the subscriptions (and getting data) from one sole /// Deals with the subscriptions (and getting data) from one
/// pubsub server. Theres no sharing of conenctions /// pubsub server. Once again, theres no sharing of conenctions
/// from this point. Every new instance of SingleServerSubscriberCore /// from this point. Every new instance of SingleServerSubscriberCore
/// will open its new dedicated connection to the serverURL that /// will open its new dedicated connection to the serverURL that
/// its concerned with (passed as a conctructor arg). /// its concerned with (passed as a conctructor arg).
/// </summary> /// </summary>
internal interface ISingleServerSubscriberCore : IDisposable { internal interface ISingleServerSubscriberCore : IDisposable {
IList<PubSubUpdate> get_updates(int timeout); IList<PubSubUpdate> getUpdates(int timeout);
event PubSubConnectionChange onPubSubConnectionChange; event PubSubConnectionChange onPubSubConnectionChange;
void subscribe(params string[] tags); void subscribe(params string[] tags);
void unsubscribe(params string[] tags); void unsubscribe(params string[] tags);
} }
internal class SingleServerSubscriberCore : IDisposable, ISingleServerSubscriberCore { internal class SingleServerSubscriberCore : IDisposable, ISingleServerSubscriberCore {
private readonly IPubSubServerConnection _connection; private HighPerformanceTimer myTimer = new HighPerformanceTimer();
private HashSet<string> _tags = new HashSet<string>(); private PubSubServerConnectionFactory myConnectionFactory;
private IPubSubServerConnection myConnection;
private List<string> myTags = new List<string>();
public event PubSubConnectionChange onPubSubConnectionChange; public event PubSubConnectionChange onPubSubConnectionChange;
public SingleServerSubscriberCore(string server_url, PubSubServerConnectionFactory connection_factory) { public SingleServerSubscriberCore(string serverURL, PubSubServerConnectionFactory connectionFactory) {
_connection = connection_factory.getConnection(server_url); myConnectionFactory = connectionFactory;
_connection.register_connection_callback(new PubSubServerConnectionStatusChange(_handle_server_reboot)); myConnection = myConnectionFactory.getConnection(serverURL);
myConnection.registerCallback(new ConnectionStatusChange(myHandleServerReboot));
} }
// gets as many data items that it can get in public IList<PubSubUpdate> getUpdates() {
// timeout milliseconds return getUpdates(1);
public IList<PubSubUpdate> get_updates(int timeout) { }
_connection.doWork();
List<PubSubUpdate> rv = new List<PubSubUpdate>();
IList<byte[]> data_items = _connection.get_updates(new TimeSpan(0, 0, 0, 0, timeout), Int32.MaxValue);
foreach (byte[] data in data_items) {
process_message(data, ref rv);
}
return rv;
/// <summary>
/// gets as many data items that it can get in
/// timeout milliseconds
/// </summary>
/// <param name="timeout"></param>
/// <returns></returns>
public IList<PubSubUpdate> getUpdates(int timeout) {
myConnection.doWork();
List<PubSubUpdate> retVal = new List<PubSubUpdate>();
IList<byte[]> dataItems = myConnection.getUpdates(new TimeSpan(0, 0, 0, 0, timeout), Int32.MaxValue);
foreach (byte[] data in dataItems) {
processMessage(data, retVal);
}
return retVal;
}
public List<string> getSubscribedTags() {
return myTags;
} }
public void subscribe(params string[] tags) { public void subscribe(params string[] tags) {
HashSet<string> this_call_tags = new HashSet<string>();
foreach (string tag in tags) { foreach (string tag in tags) {
if (_tags.Contains(tag)) { if (myTags.Contains(tag)) {
bts.Log.Gen.warn($"SingleServerSubscriberCore.subscribe tag: {tag} has already " + string errMsg = String.Format("tag {0} was already subscribed from server {1}",
$"been subscribed to server: {_connection.server_url}"); tag, myConnection.serverURL);
// BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning);
} else { } else {
_tags.Add(tag); myTags.Add(tag);
this_call_tags.Add(tag);
} }
} }
_subscribe(this_call_tags.ToArray()); mySubscribe(tags);
} }
public void unsubscribe(params string[] tags) { public void unsubscribe(params string[] tags) {
List<string> unsub_tags = new List<string>(); List<string> unsubTags = new List<string>();
foreach (string tag in tags) { foreach (string tag in tags) {
if (_tags.Contains(tag)) { if (myTags.Contains(tag)) {
_tags.Remove(tag); myTags.Remove(tag);
unsub_tags.Add(tag); unsubTags.Add(tag);
} else { } else {
bts.Log.Gen.warn($"tag {tag} is being unsubscribed from server {_connection.server_url}, eventhough " + string errMsg = String.Format("tag {0} is being unsubscribed from server {1}, eventhough " +
"it was never subscribed"); "it was never subscribed", tag, myConnection.serverURL);
// BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning);
} }
} }
UnSubMetaData unsub = new UnSubMetaData(_connection.client_url, unsub_tags.ToArray()); UnSubMetaData unsub = new UnSubMetaData(myConnection.clientURL, unsubTags.ToArray());
Stream str = new MemoryStream(); Stream str = new MemoryStream();
new MetaData(unsub).save(str); new MetaData(unsub).save(str);
str.Position = 0; str.Position = 0;
// this is where data is actually sent out // this is where data is actually sent out
_connection.send(str); myConnection.send(str);
} }
#region IDisposable Members #region IDisposable Members
@@ -86,40 +99,44 @@ namespace BCG {
public void Dispose() { public void Dispose() {
///remove the subscriber ///remove the subscriber
//myConnection.send(new PubSubMessage(new MetaData(new RemoveSubscriberMetaData(myConnection.clientURL))).toStream()); //myConnection.send(new PubSubMessage(new MetaData(new RemoveSubscriberMetaData(myConnection.clientURL))).toStream());
_connection.Dispose(); myConnection.Dispose();
//clear the list of tags you've subscribed to. This way, if you're ever used again, //clear the list of tags you've subscribed to. This way, if you're ever used again,
//you have to send all the subscription metadata out again. //you have to send all the subscription metadata out again.
_tags.Clear(); myTags.Clear();
} }
#endregion #endregion
// The only place where a tag is subsribed from the Pubsub server. the subscriptions are wrapped into SubMetaData, private void mySubscribe(params string[] tags) {
// which has a to_stream method SubMetaData sub = new SubMetaData(myConnection.clientURL, tags);
private void _subscribe(params string[] tags) {
SubMetaData sub = new SubMetaData(_connection.client_url, tags);
Stream str = new MemoryStream(); Stream str = new MemoryStream();
new MetaData(sub).save(str); new MetaData(sub).save(str);
str.Position = 0; str.Position = 0;
_connection.send(str); myConnection.send(str);
} }
private void _handle_server_reboot(string server_url, bool connection) { private void myHandleServerReboot(string serverURL, bool connection) {
bts.Log.Gen.info($"SingleServerSubscriberCore._handle_server_reboot connection: {connection} server_url: {server_url}");
if (connection) { if (connection) {
_subscribe(_tags.ToArray()); //ie connection established again, so has to
//resubscribe to all the tags
// BlueEventLog.logEvent("Connection to " + serverURL + " established....resubscribing to all the tags",
// PS3LogLevels.disconnection);
mySubscribe(myTags.ToArray());
} else { } else {
bts.Log.Gen.warn($"SingleServerSubscriberCore._handle_server_reboot connection to {server_url} broke....attempting to reconnect..."); // BlueEventLog.logEvent("Connection to " + serverURL + " broke....attempting to reconnect...",
// PS3LogLevels.disconnection);
} }
//let anyone watching you know that your connection has changed. //let anyone watching you know that your connection has changed.
if (onPubSubConnectionChange != null) if (onPubSubConnectionChange != null)
onPubSubConnectionChange(connection, server_url); onPubSubConnectionChange(connection, serverURL);
} }
// processes one message internal static void processMessage(byte[] data, List<PubSubUpdate> updates) {
internal static void process_message(byte[] data, ref List<PubSubUpdate> updates) { bts.Log.Gen.info($"SingleServerSubscriberCore.processMessage data_len: {data.Length}");
bts.Log.Gen.info($"SingleServerSubscriberCore.process_message data_len: {data.Length}");
if (data != null && data.Length > 0) { if (data != null && data.Length > 0) {
MemoryStream str = new MemoryStream(data); MemoryStream str = new MemoryStream(data);
MetaData md = new MetaData(str); MetaData md = new MetaData(str);
@@ -139,8 +156,9 @@ namespace BCG {
if (p.isCompressed) { if (p.isCompressed) {
long compressedSize = bodyStream.Length - bodyStream.Position; long compressedSize = bodyStream.Length - bodyStream.Position;
if (compressedSize == 0) { if (compressedSize == 0) {
string error_msg = "Error: tag " + p.tag + " had compressed size of 0 bytes. This shouldnt' happen. Not decompressing tag or sending out update."; string errorMsg = "Error: tag " + p.tag + " had compressed size of 0 bytes. This shouldnt' happen. Not decompressing tag or sending out update.";
bts.Log.Gen.error(error_msg); Console.WriteLine(errorMsg);
//BlueEventLog.logEvent(errorMsg, BlueEventLevel.Error);
return; return;
} }
realBody = StreamDecompressor.decompressStream(bodyStream); realBody = StreamDecompressor.decompressStream(bodyStream);
@@ -156,9 +174,8 @@ namespace BCG {
bodyStream.Read(trConstituent, 0, trConstituent.Length); bodyStream.Read(trConstituent, 0, trConstituent.Length);
updates.Add(new PubSubUpdate(pub.tag, pub.type, new MemoryStream(trConstituent))); updates.Add(new PubSubUpdate(pub.tag, pub.type, new MemoryStream(trConstituent)));
} }
} else { } else { }
bts.Log.assert_d(false, $"Unknown message received: {md} only Pub and TransactionMetaData are legal"); //BlueEventLog.logAssert(false, "Unknown message received: " + md.ToString() + " , only Pub and TransactionMetaData are legal");
}
} }
} }
} }
@@ -387,18 +387,18 @@ namespace BCG {
SharpZipCompressor Kompressor = new SharpZipCompressor(); SharpZipCompressor Kompressor = new SharpZipCompressor();
Kompressor.compress(content, contentToPublish); Kompressor.compress(content, contentToPublish);
contentToPublish.Position = 0; contentToPublish.Position = 0;
pub_md = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, serverConn.server_url, ClientInfo.thisClient.processString); pub_md = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString);
if (PubSub3Config.instance.shouldReportTagsCompressionRatio(tag)) { if (PubSub3Config.instance.shouldReportTagsCompressionRatio(tag)) {
string msg = String.Format("Compressed tag {0} from {1} to {2} bytes, for a ratio of {3}.", tag, content.Length, contentToPublish.Length, (float)content.Length / (float)contentToPublish.Length); string msg = String.Format("Compressed tag {0} from {1} to {2} bytes, for a ratio of {3}.", tag, content.Length, contentToPublish.Length, (float)content.Length / (float)contentToPublish.Length);
// BlueEventLog.logEvent(msg, PS3LogLevels.compressionRatio); // BlueEventLog.logEvent(msg, PS3LogLevels.compressionRatio);
} }
} else } else
pub_md = new PubMetaData(tagType, tag, (int)content.Length, serverConn.server_url, ClientInfo.thisClient.processString); pub_md = new PubMetaData(tagType, tag, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString);
content.Position = initPos; content.Position = initPos;
if (CrushedTagManager.instance.is_tag_crushable(tag)) { if (CrushedTagManager.instance.is_tag_crushable(tag)) {
string refId = String.Empty; string refId = String.Empty;
Console.WriteLine("StaticPublisherCore.mySend serverConn.sendWithConfirmation tag: " + tag + " server: " + serverConn.server_url); Console.WriteLine("StaticPublisherCore.mySend serverConn.sendWithConfirmation tag: " + tag + " server: " + serverConn.serverURL);
// this is where the data is actually sent out (there are multiple sends) // this is where the data is actually sent out (there are multiple sends)
PubSubMessage ps_msg = new PubSubMessage(new MetaData(pub_md), contentToPublish); PubSubMessage ps_msg = new PubSubMessage(new MetaData(pub_md), contentToPublish);
//Console.WriteLine($"serverConn.send_with_confirmation metadata: {ps_msg.meta_data}"); //Console.WriteLine($"serverConn.send_with_confirmation metadata: {ps_msg.meta_data}");
@@ -428,8 +428,8 @@ namespace BCG {
myTagToSender.Add(tag, sender); myTagToSender.Add(tag, sender);
if (!mySenders.Contains(sender)) { if (!mySenders.Contains(sender)) {
mySenders.Add(sender); mySenders.Add(sender);
sender.register_connection_callback(new PubSubServerConnectionStatusChange(myHandleServerReboot)); sender.registerCallback(new ConnectionStatusChange(myHandleServerReboot));
sender.register_send_confirmation_callback(new SendConfirmationDelegate(myOnSendConfirm)); sender.registerSendConfirmationCallback(new SendConfirmationDelegate(myOnSendConfirm));
} }
} }
@@ -443,7 +443,7 @@ namespace BCG {
if (connection) { if (connection) {
string thisPublisherInfo = ClientInfo.thisClient.processString; string thisPublisherInfo = ClientInfo.thisClient.processString;
foreach (KeyValuePair<string, IPubSubServerConnection> p in myTagToSender) { foreach (KeyValuePair<string, IPubSubServerConnection> p in myTagToSender) {
if (p.Value.server_url == serverURL) { if (p.Value.serverURL == serverURL) {
string tagType; string tagType;
string tag = p.Key; string tag = p.Key;
myCrushableTagsWaitingOnAck.Clear(); myCrushableTagsWaitingOnAck.Clear();
@@ -80,7 +80,7 @@ namespace BCG {
int timeShare = timeout / myServerURLToSubCore.Count; int timeShare = timeout / myServerURLToSubCore.Count;
List<PubSubUpdate> retVal = new List<PubSubUpdate>(); List<PubSubUpdate> retVal = new List<PubSubUpdate>();
foreach (ISingleServerSubscriberCore subCore in myServerURLToSubCore.Values) foreach (ISingleServerSubscriberCore subCore in myServerURLToSubCore.Values)
retVal.AddRange(subCore.get_updates(timeShare)); retVal.AddRange(subCore.getUpdates(timeShare));
_log_n_updates(retVal); _log_n_updates(retVal);
myTimer.stop(); myTimer.stop();
//if (myTimer.lastIntervalTime > 10) //if (myTimer.lastIntervalTime > 10)
Loaded 100 of 453 files, more files were not shown because too many files have changed in this diff. Show more