Reading big xml file - xml-parsing

Question

I need to read big xml file in Microsoft Visual FoxPro 9 SP2 desktop application.
Created cursor result which contains desired columns and convert.xsl file to transform xml to format readably by xmltocursor.
Tried
source = CreateObject('MSXML.Domdocument')
stylesheet = CreateObject('MSXML.Domdocument')
resultDoc = CreateObject('MSXML.Domdocument')
resultDoc.validateOnParse = .t.
stylesheet.load('convert.xsl')
source.load( 'bigxml.xml' )
source.transformNodeToObject(stylesheet, #resultDoc)
* Exception code=E06D7363
IF XMLToCursor(resultDoc.xml, 'result', 8192 )=0
But got
Fatal exception
Exception code=E06D7363
at line
IF XMLToCursor(resultDoc.xml, 'result', 8192 )=0
and application terminates.
How to convert big xml file to cursor ?
XSL is:
<?xml version="1.0" encoding="ISO-8859-1" standalone="no" ?>
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
<xsl:output method="xml" version="1.0" encoding="UTF-8" standalone="yes"/>
<!-- this replaces root tag with VFPData which is required from CursorToXML to work properly -->
<xsl:template match="/">
<xsl:element name="VFPData">
<xsl:apply-templates/>
</xsl:element>
</xsl:template>
<!-- this will denormalize XML data -->
<xsl:template match="/ettevotjad/ettevotja">
<xsl:element name="Document-Ariregister">
<xsl:element name="nimi">
<xsl:value-of select="nimi"/>
</xsl:element>
<xsl:element name="ariregistr">
<xsl:value-of select="ariregistri_kood"/>
</xsl:element>
<xsl:element name="kmkr_nr">
<xsl:value-of select="kmkr_nr"/>
</xsl:element>
<xsl:element name="ettevotja_">
<xsl:value-of select="ettevotja_staatus"/>
</xsl:element>
<xsl:element name="asukoht_et">
<xsl:value-of select="ettevotja_aadress/asukoht_ettevotja_aadressis"/>
</xsl:element>
<xsl:element name="asukoha_e2">
<xsl:value-of select="ettevotja_aadress/asukoha_ehak_tekstina"/>
</xsl:element>
<xsl:element name="indeks_ett">
<xsl:value-of select="ettevotja_aadress/indeks_ettevotja_aadressis"/>
</xsl:element>
</xsl:element>
</xsl:template>
<!-- to ommit nodes data -->
<xsl:template match="text()">
</xsl:template>
<!-- to work over every node -->
<xsl:template match="*">
<xsl:apply-templates/>
</xsl:template>
</xsl:stylesheet>

Answer 1

Here is a solution using mxml.6.0., no need for xs file:
* requires msxml.6.0! ( https://www.microsoft.com/es-es/download/details.aspx?id=3988 )
*
Parameters fileName,xPath
fileName = "D:\Data\Xml\ettevotja_rekvisiidid_2021-01-27.xml"
xPath = '/ettevotjad'
xPath = RTRIM(m.xPath,1,'/')
Private All
oxml = Createobject('msxml2.domDocument.6.0')
Wait 'loading document...' Window Nowait
With oXml As msxml.DOMDocument
.Load(Fullpath(m.fileName))
Wait Clear
If .parseError.errorCode # 0
Messagebox(.parseError.reason,16)
Return
Endif
Endwith
Close Tables All
nsteps = 500
tini = Seconds()
Set Escape On
x = 1
DO WHILE .t.
subset = oXml.selectNodes(Textmerge(m.xPath+'/*[ position() >= <<m.x>> and position() < <<m.x+m.nsteps>> ]'))
IF subset.length = 0
EXIT
ENDIF
cXml = ''
For Y = 1 To subset.Length
m.cXml = m.cXml + subset.Item(m.y-1).XML
ENDFOR
x = m.x + subset.Length
Wait Textmerge('<<m.x-1>> records << (m.x-1) / (SECONDS() - m.tini) >> records/sec ') Window nowait
cXml = '<xml encoding="windows-1252">'+m.cXml+'</xml>'
Xmltocursor(m.cXml,'xmlImport',Iif(Used('xmlImport'),4+8192,0))
ENDDO
Browse Normal Font 'consolas,8'

Answer 2

I like Marco's solution for at least it is in VFP. However, having big XML files like this, I would also care about performance. I tried with Marco's code and it took 80+ seconds to complete and may need some work if you really want those fields separated as in your XSL.
I tried, reading from that XML, transforming based on your XSL and writing to an SQLite database using C#. It took 7.2 seconds and those address lines were already separated.
Here is C# code if you would use anyway:
void Main()
{
string dataFile = #"d:\Andrus\bigdata.sqlite";
string constr = $"Data Source={dataFile}";
Stopwatch sw = new Stopwatch();
sw.Start();
if (!File.Exists(dataFile))
{
CreateDatabase(constr);
}
InsertData(constr, #"d:\Andrus\bigxml.xml");
sw.Stop();
sw.Dump("Duration");
}
void InsertData(string connectionString, string xmlFile)
{
using (SQLiteConnection conn = new SQLiteConnection(connectionString))
using (SQLiteCommand sql = new SQLiteCommand(#"insert into MyData
(nimi, ariregistr, asukoht_et, asukoha_e2, indeks_ett, kmkr_nr)
values
(?,?,?,?,?,?)", conn))
{
sql.Parameters.AddWithValue("nimi", "");
sql.Parameters.AddWithValue("ariregistr", 0);
sql.Parameters.AddWithValue("asukoht_et", "");
sql.Parameters.AddWithValue("asukoha_e2", "");
sql.Parameters.AddWithValue("indeks_ett", "");
sql.Parameters.AddWithValue("kmkr_nr", "");
conn.Open();
SQLiteTransaction transaction = conn.BeginTransaction();
XmlReaderSettings settings = new XmlReaderSettings();
settings.IgnoreWhitespace = true;
using (XmlReader r = XmlReader.Create(xmlFile, settings))
{
r.MoveToContent();
r.ReadStartElement("ettevotjad");
while (r.Name == "ettevotja")
{
XElement x = (XElement)XNode.ReadFrom(r);
string nimi = (string)x.Element("nimi");
int? ariregistr = (int?)x.Element("ariregistri_kood");
string asukoht_et = (string)x.Element("ettevotja_aadress").Element("asukoht_ettevotja_aadressis");
string asukoha_e2 = (string)x.Element("ettevotja_aadress").Element("asukoha_ehak_tekstina");
string indeks_ett = (string)x.Element("ettevotja_aadress").Element("indeks_ettevotja_aadressis");
string kmkr_nr = (string)x.Element("kmkr_nr");
///
sql.Parameters["nimi"].Value = nimi;
sql.Parameters["ariregistr"].Value = ariregistr;
sql.Parameters["asukoht_et"].Value = asukoht_et;
sql.Parameters["asukoha_e2"].Value = asukoha_e2;
sql.Parameters["indeks_ett"].Value = indeks_ett;
sql.Parameters["kmkr_nr"].Value = kmkr_nr;
sql.ExecuteNonQuery();
}
r.ReadEndElement();
}
transaction.Commit();
conn.Close();
}
}
void CreateDatabase(string connectionString)
{
using (SQLiteConnection conn = new SQLiteConnection(connectionString))
using (SQLiteCommand sql = new SQLiteCommand())
{
sql.CommandText = #"create table MyData (
nimi text,
ariregistr number null,
asukoht_et text,
asukoha_e2 text,
indeks_ett text,
kmkr_nr text
)";
sql.Connection = conn;
conn.Open();
sql.ExecuteNonQuery();
conn.Close();
}
}
PS: Code is run directly inside LinqPad.