Parsing mean temperature from weather web site HTML - regex

Question

Hi I want to use VBA to pull data from weather web site. What I'm trying to do is to get number 6 from this HTML code:
</tr>
<tr>
<td class="indent"><span>Temperatura średnia</span></td>
<td>
<span class="wx-data"><span class="wx-value">6</span><span class="wx-unit"> ° C</span></span>
</td>
<td>
-
</td>
<td> </td>
</tr>
<tr>
<td class="indent"><span>Temperatura maksymalna</span></td>
<td>
<span class="wx-data"><span class="wx-value">7</span><span class="wx-unit"> ° C</span></span>
</td>
<td>
<span class="wx-data"><span class="wx-value">8</span><span class="wx-unit"> ° C</span></span>
</td>
I tried code like this:
Private Sub CommandButton1_Click()
Dim IE As Object
' Create InternetExplorer Object
Set IE = CreateObject("InternetExplorer.Application")
' You can uncoment Next line To see form results
IE.Visible = False
' URL to get data from
IE.Navigate "https://www.wunderground.com/history/airport/EPGD/2016/10/24/DailyHistory.html?req_city=Pruszcz%20Gdanski&req_statename=Polska&reqdb.zip=00000&reqdb.magic=86&reqdb.wmo=12140"
' Statusbar
Application.StatusBar = "Loading, Please wait..."
' Wait while IE loading...
Do While IE.Busy
Application.Wait DateAdd("s", 1, Now)
Loop
Application.StatusBar = "Searching for value. Please wait..."
Dim dd As String
dd = IE.Document.getElementsByClassName("Temperatura średnia")(0).innerText
MsgBox dd
' Show IE
IE.Visible = True
' Clean up
Set IE = Nothing
Application.StatusBar = ""
End Sub
Without any result (the code does nothing). I will appreciate any help.

Answer 1

Here is the example using XHR and RegEx to retrieve all table data from the webpage:
Option Explicit
Sub ExtractDataWunderground()
Dim aResult() As String
Dim sContent As String
Dim i As Long
Dim j As Long
' retrieve html content
With CreateObject("MSXML2.XMLHTTP")
.Open "GET", "https://www.wunderground.com/history/airport/EPGD/2016/10/24/DailyHistory.html", False
.Send
sContent = .ResponseText
End With
' parse with regex
With CreateObject("VBScript.RegExp")
.Global = True
.MultiLine = True
.IgnoreCase = True
' minor html simplification
.Pattern = "<span[^>]*>|</span>|[\r\n\t]*"
sContent = .Replace(sContent, "")
' match each table row
.Pattern = "<tr><td class=""indent"">(.*?)</td><td>(.*?)</td><td>(.*?)</td><td>(.*?)</td></tr>"
With .Execute(sContent)
ReDim aResult(1 To .Count, 1 To 4)
' each row
For i = 1 To .Count
With .Item(i - 1)
' each cell
For j = 1 To 4
aResult(i, j) = DecodeHTMLEntities(.SubMatches(j - 1))
Next
End With
Next
End With
End With
' output result
Cells.Delete
Output Cells(1, 1), aResult
MsgBox "Completed"
End Sub
Function DecodeHTMLEntities(sText As String) As String
Static oHtmlfile As Object
Static oDiv As Object
If oHtmlfile Is Nothing Then
Set oHtmlfile = CreateObject("htmlfile")
oHtmlfile.Open
Set oDiv = oHtmlfile.createElement("div")
End If
oDiv.innerHTML = sText
DecodeHTMLEntities = oDiv.innerText
End Function
Sub Output(oDstRng As Range, aCells As Variant)
With oDstRng
.Parent.Select
With .Resize( _
UBound(aCells, 1) - LBound(aCells, 1) + 1, _
UBound(aCells, 2) - LBound(aCells, 2) + 1 _
)
.NumberFormat = "#"
.Value = aCells
.Columns.AutoFit
End With
End With
End Sub
The output is as follows for me:
To extract the mean temperature only you can get the value from the first match having 0 index, since the mean temperature is in the first row of the table:
Sub ExtractMeanTempWunderground()
Dim sContent As String
' retrieve html content
With CreateObject("MSXML2.XMLHTTP")
.Open "GET", "https://www.wunderground.com/history/airport/EPGD/2016/10/24/DailyHistory.html", False
.Send
sContent = .ResponseText
End With
' parse with regex
With CreateObject("VBScript.RegExp")
.Global = True
.MultiLine = True
.IgnoreCase = True
' minor html simplification
.Pattern = "<span[^>]*>|</span>|[\r\n\t]*"
sContent = .Replace(sContent, "")
' match each table row
.Pattern = "<tr><td class=""indent"">.*?</td><td>(.*?)</td><td>.*?</td><td>.*?</td></tr>"
With .Execute(sContent)
If .Count = 15 Then
' get the first row value only
MsgBox DecodeHTMLEntities(.Item(0).SubMatches(0))
Else
MsgBox "Data structure inconsistence detected"
End If
End With
End With
End Sub
Function DecodeHTMLEntities(sText As String) As String
Static oHtmlfile As Object
Static oDiv As Object
If oHtmlfile Is Nothing Then
Set oHtmlfile = CreateObject("htmlfile")
oHtmlfile.Open
Set oDiv = oHtmlfile.createElement("div")
End If
oDiv.innerHTML = sText
DecodeHTMLEntities = oDiv.innerText
End Function
Note, such methods will work until the webpage structure is changed.