out >> m_Conscientia; » RegEx

Parsing HTML tables into System.Data.DataTable

Alex Peck — Thu, 06 May 2010 20:50:08 +0000

What follows is a quick and dirty class I made to parse HTML tables into DataTables. As usual, it is the result of internet search/run/bug fix/refactor.

In use, it looks little like this:

WebClient client = new WebClient();
string html = client.DownloadString(@"http://www.table.co.uk");
DataSet dataSet = HtmlTableParser.Parse(html);

Here is the implementation. It’s not optimised for runtime performance, but it works.

/// 
/// HtmlTableParser parses the contents of an html string into a System.Data DataSet or DataTable.
/// 
public class HtmlTableParser
{
    private const RegexOptions ExpressionOptions = RegexOptions.Singleline | RegexOptions.Multiline | RegexOptions.IgnoreCase;
 
    private const string CommentPattern = "";
    private const string TablePattern = "]*>(.*?)

"; private const string HeaderPattern = "]*>(.*?)"; private const string RowPattern = "]*>(.*?)"; private const string CellPattern = "]*>(.*?)"; ///

/// Given an HTML string containing n table tables, parse them into a DataSet containing n DataTables. ///

/// An HTML string containing n HTML tables /// A DataSet containing a DataTable for each HTML table in the input HTML public static DataSet ParseDataSet(string html) { DataSet dataSet = new DataSet(); MatchCollection tableMatches = Regex.Matches( WithoutComments(html), TablePattern, ExpressionOptions); foreach (Match tableMatch in tableMatches) { dataSet.Tables.Add(ParseTable(tableMatch.Value)); } return dataSet; } ///

/// Given an HTML string containing a single table, parse that table to form a DataTable. ///

/// An HTML string containing a single HTML table /// A DataTable which matches the input HTML table public static DataTable ParseTable(string tableHtml) { string tableHtmlWithoutComments = WithoutComments(tableHtml); DataTable dataTable = new DataTable(); MatchCollection rowMatches = Regex.Matches( tableHtmlWithoutComments, RowPattern, ExpressionOptions); dataTable.Columns.AddRange(tableHtmlWithoutComments.Contains(") ? ParseColumns(tableHtml) : GenerateColumns(rowMatches)); ParseRows(rowMatches, dataTable); return dataTable; } ///

/// Strip comments from an HTML stirng ///

/// An HTML string potentially containing comments /// The input HTML string with comments removed private static string WithoutComments(string html) { return Regex.Replace(html, CommentPattern, string.Empty, ExpressionOptions); } ///

/// Add a row to the input DataTable for each row match in the input MatchCollection ///

/// A collection of all the rows to add to the DataTable /// The DataTable to which we add rows private static void ParseRows(MatchCollection rowMatches, DataTable dataTable) { foreach (Match rowMatch in rowMatches) { // if the row contains header tags don't use it - it is a header not a row if (!rowMatch.Value.Contains(")) { DataRow dataRow = dataTable.NewRow(); MatchCollection cellMatches = Regex.Matches( rowMatch.Value, CellPattern, ExpressionOptions); for (int columnIndex = 0; columnIndex < cellMatches.Count; columnIndex++) { dataRow[columnIndex] = cellMatches[columnIndex].Groups[1].ToString(); } dataTable.Rows.Add(dataRow); } } } ///

/// Given a string containing an HTML table, parse the header cells to create a set of DataColumns /// which define the columns in a DataTable. ///

/// An HTML string containing a single HTML table /// A set of DataColumns based on the HTML table header cells private static DataColumn[] ParseColumns(string tableHtml) { MatchCollection headerMatches = Regex.Matches( tableHtml, HeaderPattern, ExpressionOptions); return (from Match headerMatch in headerMatches select new DataColumn(headerMatch.Groups[1].ToString())).ToArray(); } ///

/// For tables which do not specify header cells we must generate DataColumns based on the number /// of cells in a row (we assume all rows have the same number of cells). ///

/// A collection of all the rows in the HTML table we wish to generate columns for /// A set of DataColumns based on the number of celss in the first row of the input HTML table private static DataColumn[] GenerateColumns(MatchCollection rowMatches) { int columnCount = Regex.Matches( rowMatches[0].ToString(), CellPattern, ExpressionOptions).Count; return (from index in Enumerable.Range(0, columnCount) select new DataColumn("Column " + Convert.ToString(index))).ToArray(); } }

As always, here are the tests. They yield 100% coverage but I still need to add some asserts on the column names.

/// 
/// Tests for the HtmlTableParser class
/// 
[TestClass]
public class ParserTest
{
    private TestContext testContextInstance;
 
    /// 
    /// Verify that HtmlTableParser can parse an HTML file containing a single table. The
    /// test file includes a commented out table which should be ignored. Note some tags use
    /// attributes (we test we can parse tags with and without attributes).
    /// 
    [TestMethod]
    [DeploymentItem(@"data\singleTable.txt")]
    public void TestParseSingleTable()
    {
        string html = File.ReadAllText("singleTable.txt");
        DataTable table = HtmlTableParser.ParseTable(html);
 
        AssertTable(GetExpectedData(), table);
    }
 
    /// 
    /// Verify that HtmlTableParser can parse an HTML file containing multiple tables. The
    /// test file includes a commented out table which should be ignored. The test file
    /// contains tables both with and without headers.
    /// 
    [TestMethod]
    [DeploymentItem(@"data\multipleTables.txt")]
    public void TestParseMultipleTables()
    {
        string html = File.ReadAllText("multipleTables.txt");
        DataSet dataSet = HtmlTableParser.ParseDataSet(html);
        Assert.AreEqual(3, dataSet.Tables.Count);
 
        var expected = GetExpectedData();
 
        foreach (DataTable table in dataSet.Tables)
        {
            AssertTable(expected, table);
        }
    }
 
    private static string[][] GetExpectedData()
    {
        return new[]
        {
            new[] { "row 1, cell 1", "row 1, cell 2" },
            new[] { "row 2, cell 1", "row 2, cell 2" }
        };
    }
 
    private static void AssertTable(string[][] expected, DataTable table)
    {
        Assert.AreEqual(expected.Count(), table.Rows.Count, "Table did not contain the expected number of rows");
 
        for (int i = 0; i < expected.Count(); i++)
        {
            for (int j = 0; j < expected[i].Count(); j++)
            {
                string actualElement = (table.Rows[i][j] as string).Trim();
                string expectedElement = expected[i][j];
 
                Assert.AreEqual<string>(expectedElement, actualElement, "Table did not contain the expected element");
            }
        }
    }
}

These are the test files, which are just some basic HMTL.




      
    
      Title

 

      
 
      
      
            
                  Heading 1
                  Heading 2
            
            
                  row 1, cell 1
                  row 1, cell 2
            
            
                  row 2, cell 1
                  row 2, cell 2




      
    
      Title

 

      
      
            
                  row 1, cell 1
                  row 1, cell 2
            
            
                  row 2, cell 1
                  row 2, cell 2
            
       
      
      
            
                  Heading 1
                  Heading 2
            
            
                  row 1, cell 1
                  row 1, cell 2
            
            
                  row 2, cell 1
                  row 2, cell 2
            
       
      
            
                  row 1, cell 1
                  row 1, cell 2
            
            
                  row 2, cell 1
                  row 2, cell 2

Regular Expressions

Alex Peck — Mon, 03 Aug 2009 18:17:18 +0000

Regular Expressions are used to match patterns in strings. These patterns may be particular character sequences, character classes (e.g. white space, digits, upper case etc) or combinations thereof. Patterns may be combined to form more complex patterns using repetition and alternation operators.

For a more detailed treatment, please refer to this site, which is worth visiting just to see the photograph of the author. If that’s too long winded, try the cheat sheet instead.

Regex: A Simple Example

Whilst the concept is simple, RegEx syntax is not. RegEx’s can be hard to read. This expression matches a UK postcode:

^[A-Za-z]{1,2}[\d]{1,2}([A-Za-z])?\s?[\d][A-Za-z]{2}$

This example includes several key bits of RegEx syntax. The ^ and $ characters are anchors, and specify the beginning and end. Square brackets denote a character set to match, [A-Za-z] matches upper and lower case characters in the range A-Z. Curly braces denote the number of matches, so [A]{1,2} will match the character A once or twice. The ? matches the preceeding expression zero or once, so \s? matches a white space character zero or once.

In the context of .NET, the Regex class provides static methods to test a string for a match. Here is a simple but useless example:

if (RegEx.IsMatch("pattern", @"^[aenprt]{1,7}$"))
{
   // do something...
}

Options

RegEx methods and constructors, where appropriate, provide an override where RegexOptions may be specified (e.g. compiled, case sensitive etc.). Options may be combined using a bitwise OR. You can also specify options inline within the regex pattern, by enclosing characters which map to regex options within brackets. There are two forms:

Grouping construct (?imnsx-imnsx:)
Miscelleneous construct (?imnsx-imnsx)

Prefixing a set of options with the minus sign turns them off. All the options are turned off by default. Compiled and RightToLeft may only be applied to an expression as a whole.

Grouping & Backreferences

Round brackets specify a group, and when referring back to a group it is possible to apply alternation and repitition operators. A simple example might be to search for repeating sequences of numbers preceeded by a space:

(?\s\d)\k

The (?pattern) construct is used to denote the group, then \k is used to refer back to the group. If you don’t specify a name within angle brackets, you can refer to groups by number using \ln, where n is the group number.

Extracting matches

Using the static Regex.Match method you may extract matches from an input string. The sub pattern you wish to extract should be enclosed in parentheses.

string input = "Exception: error message";
Match m = Regex.Match(input, "Exception: (.*$)");
string match1 = m.Groups[1].Value;
 
CaptureCollection captures = m.Groups[1].Captures;
Capture c = captures[0];
Console.WriteLine(string.Format("group '{0}', capture '{1}', index {2}", match1, c.Value, c.Index));
// prints: group 'error message', capture 'error message', index 11

If there is a match, the Match.Success property = true. The Match.Groups array contains the matches, indexed starting at 1. If you name the groups (using angle brackets as described above), you can also index by name. Groups comprise of capture collections, and each capture contains the match along with index and length properties.

Replacing matches

Using the static Regex.Replace method you may replace matches within the input string. This simple example shows how to use named backreferences within the replacement pattern.

string output = Regex.Replace("alex peck", @"(?\S+) (?\S+)", "${surname}, ${forename}");
// output = "peck, alex"

The replacement expression ${surname} inserts the match captured by the group (?). We could also use the expression $n, where n is an integer, to insert groups by number.

Online resources

regexlib – Regular Expression Library
regexpal – Test regular expressions in a web page (based on java)
Regulator and Regulazy are available here
reanimator – animate your regex as an automaton
.NET MSDN reference