Files
spark/src/DataStreamSuite/DataStream/Primitives/Misc/WebScrapingStream.cs
T

122 lines
4.4 KiB
C#

using System;
using System.Collections.Generic;
using System.IO;
using System.Security.Cryptography.X509Certificates;
using BCG;
using BCG.Utilities.General;
using HtmlAgilityPack;
namespace BCG {
class Program {
static void Main() {
string url = "http://quotes.toscrape.com";
List<string> quotes = ScrapeQuotes(url);
Console.WriteLine("Quotes from http://quotes.toscrape.com:");
foreach (var quote in quotes) {
Console.WriteLine(quote);
}
}
static List<string> ScrapeQuotes(string url) {
List<string> quotes = new List<string>();
// Create a new HtmlWeb instance
HtmlWeb web = new HtmlWeb();
try {
// Load the HTML document from the specified URL
HtmlDocument document = web.Load(url);
// Extract quotes from the HTML document
var quoteNodes = document.DocumentNode.SelectNodes("//span[@class='text']");
if (quoteNodes != null) {
foreach (var quoteNode in quoteNodes) {
string quote = quoteNode.InnerText.Trim();
quotes.Add(quote);
}
}
} catch (Exception ex) {
Console.WriteLine($"Error: {ex.Message}");
}
return quotes;
}
}
public class WebScapperStream : ViewStream, IWorker {
private readonly string _url;
private readonly string _xpath;
private readonly ViewStreamDatabase _vsd;
private IDictionary<string, int> _line_id_map = new Dictionary<string, int>();
private readonly IBlueTableSchema _schema;
public WebScapperStream()
:
this("http://quotes.toscrape.com", "//span[@class='text']", ViewStreamDatabase.instance) { }
public WebScapperStream(string url, string xpath)
: this(url, xpath, ViewStreamDatabase.instance) { }
public WebScapperStream(string url, string xpath, ViewStreamDatabase vsd) {
bts.Log.Gen.info($"WebScapperStream cons url: {url} xpath: {xpath}");
_schema = BlueDataUtility.form_schema($"{url}_{xpath}",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("url", TypeCode.UInt32),
Tuple.Create("line", TypeCode.String));
_url = url;
_xpath = xpath;
_vsd = vsd;
}
protected override void myOnFirstSubscriber() {
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
}
protected override void myOnLastSubscriber() {
MasterWorker.getInstance(false).removeWorker(this);
}
public void doWork() {
var lines = scrape_lines(_url, _xpath);
Func<string, int> get_id = (line) => {
int rv = 0;
if (!_line_id_map.TryGetValue(line, out rv))
_line_id_map.Add(line, _line_id_map.Count);
return rv;
};
BlueTable nrows = new BlueTable(_schema);
foreach (var l in lines) {
bts.Log.Gen.info($"line: {l}");
var id = get_id(l);
BlueRow nr = new BlueRow(_schema, get_id(l), _url, l);
nrows.addRow(nr);
}
var dp = BlueDataUtility.build_dp_inc(nrows, myLastValue as DataPacket, true);
myFireEvent(dp);
}
static List<string> scrape_lines(string url, string xpath) {
List<string> quotes = new List<string>();
// Create a new HtmlWeb instance
HtmlWeb web = new HtmlWeb();
try {
// Load the HTML document from the specified URL
HtmlDocument document = web.Load(url);
// Extract quotes from the HTML document
var quote_nodes = document.DocumentNode.SelectNodes(xpath);
if (quote_nodes != null) {
foreach (var qn in quote_nodes) {
string quote = qn.InnerText.Trim();
quotes.Add(quote);
}
}
} catch (Exception ex) {
Console.WriteLine($"Error: {ex.Message}");
}
return quotes;
}
}
}