122 lines
4.4 KiB
C#
122 lines
4.4 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Security.Cryptography.X509Certificates;
|
|
|
|
using BCG;
|
|
using BCG.Utilities.General;
|
|
|
|
using HtmlAgilityPack;
|
|
|
|
namespace BCG {
|
|
class Program {
|
|
static void Main() {
|
|
string url = "http://quotes.toscrape.com";
|
|
List<string> quotes = ScrapeQuotes(url);
|
|
|
|
Console.WriteLine("Quotes from http://quotes.toscrape.com:");
|
|
foreach (var quote in quotes) {
|
|
Console.WriteLine(quote);
|
|
}
|
|
}
|
|
|
|
static List<string> ScrapeQuotes(string url) {
|
|
List<string> quotes = new List<string>();
|
|
|
|
// Create a new HtmlWeb instance
|
|
HtmlWeb web = new HtmlWeb();
|
|
|
|
try {
|
|
// Load the HTML document from the specified URL
|
|
HtmlDocument document = web.Load(url);
|
|
|
|
// Extract quotes from the HTML document
|
|
var quoteNodes = document.DocumentNode.SelectNodes("//span[@class='text']");
|
|
if (quoteNodes != null) {
|
|
foreach (var quoteNode in quoteNodes) {
|
|
string quote = quoteNode.InnerText.Trim();
|
|
quotes.Add(quote);
|
|
}
|
|
}
|
|
} catch (Exception ex) {
|
|
Console.WriteLine($"Error: {ex.Message}");
|
|
}
|
|
|
|
return quotes;
|
|
}
|
|
}
|
|
|
|
public class WebScapperStream : ViewStream, IWorker {
|
|
private readonly string _url;
|
|
private readonly string _xpath;
|
|
private readonly ViewStreamDatabase _vsd;
|
|
private IDictionary<string, int> _line_id_map = new Dictionary<string, int>();
|
|
private readonly IBlueTableSchema _schema;
|
|
|
|
public WebScapperStream()
|
|
:
|
|
this("http://quotes.toscrape.com", "//span[@class='text']", ViewStreamDatabase.instance) { }
|
|
public WebScapperStream(string url, string xpath)
|
|
: this(url, xpath, ViewStreamDatabase.instance) { }
|
|
public WebScapperStream(string url, string xpath, ViewStreamDatabase vsd) {
|
|
bts.Log.Gen.info($"WebScapperStream cons url: {url} xpath: {xpath}");
|
|
_schema = BlueDataUtility.form_schema($"{url}_{xpath}",
|
|
Tuple.Create("id", TypeCode.Int32),
|
|
Tuple.Create("url", TypeCode.UInt32),
|
|
Tuple.Create("line", TypeCode.String));
|
|
_url = url;
|
|
_xpath = xpath;
|
|
_vsd = vsd;
|
|
}
|
|
|
|
protected override void myOnFirstSubscriber() {
|
|
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
|
|
}
|
|
|
|
protected override void myOnLastSubscriber() {
|
|
MasterWorker.getInstance(false).removeWorker(this);
|
|
}
|
|
|
|
public void doWork() {
|
|
var lines = scrape_lines(_url, _xpath);
|
|
Func<string, int> get_id = (line) => {
|
|
int rv = 0;
|
|
if (!_line_id_map.TryGetValue(line, out rv))
|
|
_line_id_map.Add(line, _line_id_map.Count);
|
|
return rv;
|
|
};
|
|
BlueTable nrows = new BlueTable(_schema);
|
|
foreach (var l in lines) {
|
|
bts.Log.Gen.info($"line: {l}");
|
|
var id = get_id(l);
|
|
BlueRow nr = new BlueRow(_schema, get_id(l), _url, l);
|
|
nrows.addRow(nr);
|
|
}
|
|
var dp = BlueDataUtility.build_dp_inc(nrows, myLastValue as DataPacket, true);
|
|
myFireEvent(dp);
|
|
}
|
|
|
|
static List<string> scrape_lines(string url, string xpath) {
|
|
List<string> quotes = new List<string>();
|
|
// Create a new HtmlWeb instance
|
|
HtmlWeb web = new HtmlWeb();
|
|
try {
|
|
// Load the HTML document from the specified URL
|
|
HtmlDocument document = web.Load(url);
|
|
// Extract quotes from the HTML document
|
|
var quote_nodes = document.DocumentNode.SelectNodes(xpath);
|
|
if (quote_nodes != null) {
|
|
foreach (var qn in quote_nodes) {
|
|
string quote = qn.InnerText.Trim();
|
|
quotes.Add(quote);
|
|
}
|
|
}
|
|
} catch (Exception ex) {
|
|
Console.WriteLine($"Error: {ex.Message}");
|
|
}
|
|
|
|
return quotes;
|
|
}
|
|
}
|
|
}
|