Is there a simple tool to flatten a wsdl file? - wsdl

I have nested WSDL that I want to make flat. The wsdl imports xsd files that in turn include other files and so on. I like to flatten the whole thing into one file in order to feed it into a tool that isn't capable of doing the imports/includes.
Is there a simple tool (maybe commandline) that I can use for it? I tried using xsltproc/xmllint but those don't know anything about wsdl include.

I guess I'm very late, but just in case anyone else will need it, here's a (admittedly very dirty) code snippet I arranged (by taking half-working pieces of code I found in various places and adding some fixes). This requires .NET 4.5 to work, but will flatten remote WSDL URLs nicely. It probably won't cover all cases, but it is working for me in last 2 years.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Web.Services.Discovery;
using System.Xml;
namespace FlattenXml {
internal class Program {
private static XmlDocument m_MapDoc;
private static string m_DocPath;
private static XmlDocument m_FlattenDocument = new XmlDocument();
private static string m_ServiceUri;
private static Dictionary<string, XmlElement> m_Elements = new Dictionary<string, XmlElement>();
private static void Main(string[] args) {
if (args == null || args.Length == 0) {
Program.usage();
} else {
if (args != null && args.Length == 1 && (args[0] == "-h" || args[0] == "-?")) {
Program.usage();
} else {
if (args != null && args.Length == 2) {
Program.m_DocPath = args[1];
Program.m_ServiceUri = args[0];
DiscoveryClientProtocol client = new DiscoveryClientProtocol();
client.DiscoverAny(Program.m_ServiceUri);
client.ResolveAll();
Program.m_DocPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, Program.m_DocPath);
Directory.CreateDirectory(Program.m_DocPath);
string mapFile = Path.Combine(Program.m_DocPath, "map.xml");
client.WriteAll(Program.m_DocPath, mapFile);
flattenDocuments(mapFile);
}
}
}
}
private static void usage() {
Console.WriteLine("Usage: flattenxml [Service Wsdl URL] [Output Folder]");
Console.WriteLine("Example: flattenxml http://localhost/Daenet.TrackingService/TrackingService.svc Out");
}
private static void flattenDocuments(string mapFile) {
string rootDocumentName = Program.getRootDocumentFile(mapFile);
Program.m_FlattenDocument = Program.flattenDocument(Path.Combine(Program.m_DocPath, rootDocumentName));
Program.m_FlattenDocument.Save("FlattenDocument.wsdl");
}
private static string getRootDocumentFile(string mapFile) {
XmlDocument mapDoc = new XmlDocument();
mapDoc.Load(mapFile);
Program.m_MapDoc = mapDoc;
foreach (XmlElement el in mapDoc.DocumentElement.FirstChild.OfType<XmlElement>()) {
if (el.Attributes["url"].Value == Program.m_ServiceUri || el.Attributes["url"].Value == Program.m_ServiceUri + "?wsdl") {
return el.Attributes["filename"].Value;
}
}
throw new ApplicationException("Root document annot be determined.");
}
private static XmlDocument flattenDocument(string rootDocumentName) {
XmlDocument unflattenDoc = new XmlDocument();
unflattenDoc.Load(rootDocumentName);
return Program.importElements(unflattenDoc.DocumentElement);
}
private static XmlDocument importElements(XmlElement schemaEl) {
XmlElement root = schemaEl.OwnerDocument.CreateElement("a");
Program.importElement(root, schemaEl);
XmlDocument flattenDoc = new XmlDocument();
flattenDoc.LoadXml(root.FirstChild.OuterXml);
return flattenDoc;
}
private static void importExternalElement(XmlElement typeParentEl, XmlElement typeEl) {
if (typeEl.LocalName.ToLower() == "import" || typeEl.LocalName.ToLower() == "include") {
string importingDocName = Program.getImportingDocName(typeEl);
if (importingDocName != null && !Program.m_Elements.ContainsKey(importingDocName)) {
Program.addComment(typeParentEl, typeEl);
XmlDocument unflattenDoc = new XmlDocument();
unflattenDoc.Load(Path.Combine(Program.m_DocPath, importingDocName));
Program.m_Elements.Add(importingDocName, null);
XmlElement importedEl = typeParentEl.OwnerDocument.CreateElement("a");
Program.importElement(importedEl, unflattenDoc.DocumentElement);
// B: for WSDL, inner "definitions" and "policy" elements should be skipped. For all other cases (i.e. XSD) everything should be included
if (typeParentEl.LocalName.ToLower() == "definitions" && typeParentEl.LocalName == importedEl.FirstChild.LocalName) {
Program.mergeNamespaces(typeParentEl, importedEl.FirstChild as XmlElement);
foreach (XmlElement cEl in importedEl.FirstChild.ChildNodes.OfType<XmlElement>()) {
// B: skip policy definitions from internal WSDLs
if (cEl.LocalName.ToLower() == "policy") {
continue;
}
XmlElement cEl2 = cEl.OwnerDocument.CreateElement("a");
cEl2.InnerXml = cEl.OuterXml;
typeParentEl.InsertBefore(cEl2.FirstChild, typeEl);
}
} else {
typeParentEl.InsertBefore(importedEl.FirstChild, typeEl);
}
Program.m_Elements[importingDocName] = importedEl;
}
typeParentEl.RemoveChild(typeEl);
return;
}
throw new ArgumentException();
}
private static XmlElement importElement(XmlElement typeParentEl, XmlElement typeEl) {
XmlElement resultingElement;
if (typeEl.ChildNodes.Count > 0) {
XmlElement newEl = typeParentEl.OwnerDocument.CreateElement("a");
newEl.InnerXml = typeEl.OuterXml;
if (typeEl.ChildNodes.OfType<XmlElement>().Count<XmlElement>() > 0) {
newEl.FirstChild.InnerXml = "";
foreach (XmlElement child in typeEl.ChildNodes.OfType<XmlElement>()) {
if (child.LocalName.ToLower() == "import" || child.LocalName.ToLower() == "include") {
XmlElement impEl = newEl.OwnerDocument.CreateElement("a");
impEl.InnerXml = child.OuterXml;
XmlElement x = newEl.FirstChild.AppendChild(impEl.FirstChild) as XmlElement;
Program.importExternalElement(newEl.FirstChild as XmlElement, x);
} else {
Program.importElement(newEl.FirstChild as XmlElement, child);
}
}
}
resultingElement = Program.appendElement(typeParentEl, newEl.FirstChild);
} else {
if (typeParentEl.ChildNodes.OfType<XmlElement>().Count((XmlElement e) => e.LocalName == typeEl.LocalName) >= 1) {
}
resultingElement = Program.appendElement(typeParentEl, typeEl);
}
return resultingElement;
}
private static XmlElement appendElement(XmlElement parentEl, XmlNode childNode) {
XmlElement newEl = parentEl.OwnerDocument.CreateElement("a");
newEl.InnerXml = childNode.OuterXml;
return parentEl.AppendChild(newEl.FirstChild) as XmlElement;
}
private static XmlDocument loadUnflattenDocument(string importingDoc) {
XmlDocument unflattenDoc = new XmlDocument();
unflattenDoc.Load(Path.Combine(Program.m_DocPath, importingDoc));
XmlDocument flattenDoc = new XmlDocument();
return unflattenDoc;
}
private static XmlDocument loadDocumentFromImportNode(XmlElement importEl) {
string importingDoc = Program.getImportingDocName(importEl);
XmlDocument unflattenDoc = new XmlDocument();
unflattenDoc.Load(Path.Combine(Program.m_DocPath, importingDoc));
return unflattenDoc;
}
private static string getImportingDocName(XmlElement importEl) {
return Program.getImportingDocName(Program.getImportingDocReference(importEl));
}
private static string getImportingDocReference(XmlElement importEl) {
if (importEl.LocalName.ToLower() == "import" || importEl.LocalName.ToLower() == "include") {
if (importEl.Attributes["location"] != null) {
return importEl.Attributes["location"].Value;
} else if (importEl.Attributes["schemaLocation"] != null) {
return importEl.Attributes["schemaLocation"].Value;
}
}
Console.WriteLine("Invalid node, ignored:");
Console.WriteLine(importEl.OuterXml);
return null;
}
private static string getImportingDocName(string docReference) {
if (docReference == null) return null;
// NOTE: [B] the following fix required .NET 4.5, because before .NET 4.5. the URLs were not URLEncoded, which was wrong. The extra check below allows it run with .NET 3.5 or later
var docReferenceRaw = Uri.UnescapeDataString(docReference);
XmlElement refEl = Program.m_MapDoc.DocumentElement.FirstChild.ChildNodes.OfType<XmlElement>().FirstOrDefault((XmlElement el) => el.Attributes["url"].Value == docReference || el.Attributes["url"].Value == docReferenceRaw);
if (refEl != null) {
return refEl.Attributes["filename"].Value;
}
throw new ApplicationException("Mapping error");
}
private static void addComment(XmlElement parentEl, XmlElement importEl) {
var location = Program.getImportingDocReference(importEl);
var path = location == null ? "" : new Uri(Program.getImportingDocReference(importEl)).PathAndQuery;
XmlComment commentEl = parentEl.OwnerDocument.CreateComment("Begin \"" + path + "\"");
parentEl.InsertBefore(commentEl, importEl);
XmlComment commentEl2 = parentEl.OwnerDocument.CreateComment("End \"" + path + "\"");
parentEl.InsertAfter(commentEl2, importEl);
}
private static void mergeNamespaces(XmlElement el1, XmlElement el2) {
foreach (XmlAttribute attr in new List<XmlAttribute>(el2.Attributes.OfType<XmlAttribute>())) {
if (el1.Attributes.OfType<XmlAttribute>().Count((XmlAttribute e) => e.Name == attr.Name) == 0) {
el1.Attributes.Append(attr);
}
}
}
}
}
Hope this helps.

Maybe flatten-wsdl.py would help to create singleWsdl file. It 's a small python script that will fetch WSDL from URL and flatten it by importing all referenced parts (i.e xsd:import)
You can use it like this and it will generate flatten.wsdl file
python flatten-wsdl.py http://URL/SOME-WSDL-WITH-INLINE-SCHEMA?wsdl -f flatten.wsdl

Related

Building Splittable DoFn (SDF) to list objects in GCS

I am trying to develop custom source for parallel GCS content scanning. The naive approach would be to loop through listObjects function calls:
while (...) {
Objects objects = gcsUtil.listObjects(bucket, prefix, pageToken);
pageToken = objects.getNextPageToken();
...
}
The problem is performance for the tens of millions objects.
Instead of the single thread code we can add delimiter / and submit parallel processed for each prefix found:
...
Objects objects = gcsUtil.listObjects(bucket, prefix, pageToken, "/");
for (String subPrefix : object.getPrefixes()) {
scanAsync(bucket, subPrefix);
}
...
Next idea was to try to wrap this logic in Splittable DoFn.
Choice of RestrictionTracker: I don't see how can be used any of exiting RestrictionTracker. So decided to write own. Restriction itself is basically list of prefixes to scan. tryClaim checks if there is more prefixes left and receive newly scanned to append them to current restriction. trySplit splits list of prefixes.
The problem that I faced that trySplit can be called before all subPrefixes are found. In this case current restriction may receive more work to do after it was splitted. And it seems that trySplit is being called until it returns a not null value for a given RestrictionTracker: after certain number of elements goes to the output or after 1 second via scheduler or when ProcessContext.resume() returned. This doesn't work in my case as new prefixes can be found at any time. And I can't checkpoint via return ProcessContext.resume() because if split was already done, possible work that left in current restriction will cause checkDone() function to fail.
Another problem that I suspect that I couldn't achieve parallel execution in DirectRunner. As trySplit was always called with fractionOfRemainder=0 and new RestrictionTracker was started only after current one completed its piece of work.
It would be also great to read more detailed explanation about Splittable DoFn components lifecycle. How parallel execution per element is achieved. And how and when state of RestrictionTracker can be modified.
UPD: Adding simplified code that should show intended implementation
#DoFn.BoundedPerElement
private static class ScannerDoFn extends DoFn<String, String> {
private transient GcsUtil gcsUtil;
#GetInitialRestriction
public ScannerRestriction getInitialRestriction(#Element String bucket) {
return ScannerRestriction.init(bucket);
}
#ProcessElement
public ProcessContinuation processElement(
ProcessContext c,
#Element String bucket,
RestrictionTracker<ScannerRestriction, ScannerPosition> tracker,
OutputReceiver<String> outputReceiver) {
if (gcsUtil == null) {
gcsUtil = c.getPipelineOptions().as(GcsOptions.class).getGcsUtil();
}
ScannerRestriction currentRestriction = tracker.currentRestriction();
ScannerPosition position = new ScannerPosition();
while (true) {
if (tracker.tryClaim(position)) {
if (position.completedCurrent) {
// position.clear();
// ideally I would to get checkpoint here before starting new work
return ProcessContinuation.resume();
}
try {
Objects objects = gcsUtil.listObjects(
currentRestriction.bucket,
position.currentPrefix,
position.currentPageToken,
"/");
if (objects.getItems() != null) {
for (StorageObject o : objects.getItems()) {
outputReceiver.output(o.getName());
}
}
if (objects.getPrefixes() != null) {
position.newPrefixes.addAll(objects.getPrefixes());
}
position.currentPageToken = objects.getNextPageToken();
if (position.currentPageToken == null) {
position.completedCurrent = true;
}
} catch (Throwable throwable) {
logger.error("Error during scan", throwable);
}
} else {
return ProcessContinuation.stop();
}
}
}
#NewTracker
public RestrictionTracker<ScannerRestriction, ScannerPosition> restrictionTracker(#Restriction ScannerRestriction restriction) {
return new ScannerRestrictionTracker(restriction);
}
#GetRestrictionCoder
public Coder<ScannerRestriction> getRestrictionCoder() {
return ScannerRestriction.getCoder();
}
}
public static class ScannerPosition {
private String currentPrefix;
private String currentPageToken;
private final List<String> newPrefixes;
private boolean completedCurrent;
public ScannerPosition() {
this.currentPrefix = null;
this.currentPageToken = null;
this.newPrefixes = Lists.newArrayList();
this.completedCurrent = false;
}
public void clear() {
this.currentPageToken = null;
this.currentPrefix = null;
this.completedCurrent = false;
}
}
private static class ScannerRestriction {
private final String bucket;
private final LinkedList<String> prefixes;
private ScannerRestriction(String bucket) {
this.bucket = bucket;
this.prefixes = Lists.newLinkedList();
}
public static ScannerRestriction init(String bucket) {
ScannerRestriction res = new ScannerRestriction(bucket);
res.prefixes.add("");
return res;
}
public ScannerRestriction empty() {
return new ScannerRestriction(bucket);
}
public boolean isEmpty() {
return prefixes.isEmpty();
}
public static Coder<ScannerRestriction> getCoder() {
return ScannerRestrictionCoder.INSTANCE;
}
private static class ScannerRestrictionCoder extends AtomicCoder<ScannerRestriction> {
private static final ScannerRestrictionCoder INSTANCE = new ScannerRestrictionCoder();
private final static Coder<List<String>> listCoder = ListCoder.of(StringUtf8Coder.of());
#Override
public void encode(ScannerRestriction value, OutputStream outStream) throws IOException {
NullableCoder.of(StringUtf8Coder.of()).encode(value.bucket, outStream);
listCoder.encode(value.prefixes, outStream);
}
#Override
public ScannerRestriction decode(InputStream inStream) throws IOException {
String bucket = NullableCoder.of(StringUtf8Coder.of()).decode(inStream);
List<String> prefixes = listCoder.decode(inStream);
ScannerRestriction res = new ScannerRestriction(bucket);
res.prefixes.addAll(prefixes);
return res;
}
}
}
private static class ScannerRestrictionTracker extends RestrictionTracker<ScannerRestriction, ScannerPosition> {
private ScannerRestriction restriction;
private ScannerPosition lastPosition = null;
ScannerRestrictionTracker(ScannerRestriction restriction) {
this.restriction = restriction;
}
#Override
public boolean tryClaim(ScannerPosition position) {
restriction.prefixes.addAll(position.newPrefixes);
position.newPrefixes.clear();
if (position.completedCurrent) {
// completed work for current prefix
assert lastPosition != null && lastPosition.currentPrefix.equals(position.currentPrefix);
lastPosition = null;
return true; // return true but we would need to claim again if we need to get next prefix
} else if (lastPosition != null && lastPosition.currentPrefix.equals(position.currentPrefix)) {
// proceed work for current prefix
lastPosition = position;
return true;
}
// looking for next prefix
assert position.currentPrefix == null;
assert lastPosition == null;
if (restriction.isEmpty()) {
// no work to do
return false;
}
position.currentPrefix = restriction.prefixes.poll();
lastPosition = position;
return true;
}
#Override
public ScannerRestriction currentRestriction() {
return restriction;
}
#Override
public SplitResult<ScannerRestriction> trySplit(double fractionOfRemainder) {
if (lastPosition == null && restriction.isEmpty()) {
// no work at all
return null;
}
if (lastPosition != null && restriction.isEmpty()) {
// work at the moment only at currently scanned prefix
return SplitResult.of(restriction, restriction.empty());
}
int size = restriction.prefixes.size();
int newSize = new Double(Math.round(fractionOfRemainder * size)).intValue();
if (newSize == 0) {
ScannerRestriction residual = restriction;
restriction = restriction.empty();
return SplitResult.of(restriction, residual);
}
ScannerRestriction residual = restriction.empty();
for (int i=newSize; i<=size; i++) {
residual.prefixes.add(restriction.prefixes.removeLast());
}
return SplitResult.of(restriction, residual);
}
#Override
public void checkDone() throws IllegalStateException {
if (lastPosition != null) {
throw new IllegalStateException("Called checkDone on not completed job");
}
}
#Override
public IsBounded isBounded() {
return IsBounded.UNBOUNDED;
}
}

Android MediatorLiveData to combine multiple LiveData results into single LiveData object (In Java)

I am using 2 LiveDatas from separate tables into the repository of my application. I add the two LiveDatas as sources to the CustomMediatorLiveData class which extends MediatorLiveData.
In the onChanged callback of addSource method for each LiveData, I send the values of both the LiveDatas into a method that combines both and returns a single LiveData which is set as the value for the CustomMediatorLiveData object. I am creating an object of this CustomMediatorLiveData in my Repository and passing the two LiveDatas as parameters for the constructor.
This runs and doesn't give any error but it is messing up the data within the LiveData itself.
example: If the date was originally 15th August 2020 then it can be something like 14th August 0001.
CustomMediatorLiveData:
public class CustomMediatorLiveData extends MediatorLiveData<List<Object>> {
private List<Note> notes = Collections.emptyList();
private List<RecurringTask> recurringTasks = Collections.emptyList();
public CustomMediatorLiveData(LiveData<List<Note>> liveNotes, LiveData<List<RecurringTask>> liveRecurringTasks) {
addSource(liveNotes, notes1 -> {
if (notes1 != null) {
this.notes = notes1;
}
setValue(combineData(notes,recurringTasks));
});
addSource(liveRecurringTasks, recurringTasks1 -> {
if (recurringTasks1 != null) {
this.recurringTasks = recurringTasks1;
}
setValue(combineData(notes,recurringTasks));
});
}
// This method adds the 2 lists into one and sorts them based on dates and priority.
private List<Object> combineData(List<Note> notes, List<RecurringTask> recurringTasks) {
List<Object> combinedList = new ArrayList<>();
if (notes != null && !notes.isEmpty())
combinedList.addAll(notes);
if(recurringTasks!=null && !recurringTasks.isEmpty())
combinedList.addAll(recurringTasks);
Collections.sort(combinedList, new Comparator<Object>() {
#Override
public int compare(Object o1, Object o2) {
Date d1, d2;
Note n1 = null, n2 = null;
RecurringTask r1 = null, r2 = null;
if (o1 instanceof Note && o2 instanceof Note) {
int hmm = Boolean.compare(((Note) o2).isPriority(), ((Note) o1).isPriority());
if (hmm != 0)
return hmm;
}
if (o1 instanceof Note) {
d1 = ((Note) o1).getEnd_date();
n1 = ((Note) o1);
} else {
d1 = ((RecurringTask) o1).getEnd_date();
r1 = ((RecurringTask) o1);
}
if (o2 instanceof Note) {
d2 = ((Note) o2).getEnd_date();
n2 = ((Note) o2);
} else {
d2 = ((RecurringTask) o2).getEnd_date();
r2 = ((RecurringTask) o2);
}
if (n1 != null) {
if (r2 != null) {
if (n1.isPriority()) {
return -1;
}
}
}
if (n2 != null) {
if (r1 != null) {
if (n2.isPriority()) {
return 1;
}
}
}
long l1 = d1.getTime() - d2.getTime();
if (l1 > 0) {
return 1;
} else if (l1 < 0) {
return -1;
} else {
return 0;
}
}
});
return combinedList;
}
}
Note Repository class:
public class NoteRepository {
private String DB_NAME = "db_task";
Context context;
private RecurringDao recurringDao;
private LiveData<List<RecurringTask>> upcomingRecurringTasks;
private LiveData<List<Note>> upcomingTasks;
private CustomMediatorLiveData customMediatorLiveData;
private NoteDatabase noteDatabase;
public NoteRepository(Context context) {
noteDatabase = NoteDatabase.getInstance(context);
recurringDao = noteDatabase.recurringDao();
upcomingRecurringTasks = recurringDao.getUpcomingRecurringTask();
upcomingTasks = noteDatabase.daoAccess().fetchUpcomingTasks();
this.context = context;
customMediatorLiveData = new CustomMediatorLiveData(upcomingTasks, upcomingRecurringTasks);
}
public LiveData<List<Object>> getCustomMediatorLiveData() {
return customMediatorLiveData;
}
public LiveData<List<RecurringTask>> getUpcomingRecurringTasks() {
return upcomingRecurringTasks;
}
public LiveData<List<Note>> fetchUpcomingTasks() {
return NoteDatabase.getInstance(context).daoAccess().fetchUpcomingTasks();
}
}
I have tried using the MediatorLiveData object and add the two LiveData sources to it in the repository itself and the same issue persists.
What is the correct way to implement this? How to combine 2 LiveDatas into a single LiveData that can be observed.

Is this the best way to get all elements & attributes in an XML file using Saxon?

Our program displays a tree control showing the metadata structure of the XML file they are using as a datasource. So it displays all elements & attributes in use in the XML file, like this:
Employees
Employee
FirstName
LastName
Orders
Order
OrderId
For the case where the user does not pass us a XSD file, we need to walk the XML file and build up the metadata structure.
The full code for this is at SaxonQuestions.zip, TestBuildTree.java and is also listed below.
I am concerned that my code is not the most efficient, or maybe even wrong. It works, but it works on the 3 XML files I tested it on. My questions are:
What is the best way to get the root element from the DOM? Is it walking the children of the DOM root node?
What is the best way to determine if an element has data (as opposed to just child elements)? The best I could come up with throws an exception if not and I don't think code executing the happy path should throw an exception.
What is the best way to get the class of the data held in an element or attribute? Is it: ((XdmAtomicValue)((XdmNode)currentNode).getTypedValue()).getValue().getClass();
Is the best way to walk all the nodes to use XdmNode.axisIterator? And to do so as I have in this code?
TestBuildTree.java
import net.sf.saxon.s9api.*;
import java.io.File;
import java.io.FileInputStream;
import java.util.ArrayList;
import java.util.List;
public class TestBuildTree {
public static void main(String[] args) throws Exception {
XmlDatasource datasource = new XmlDatasource(
new FileInputStream(new File("files", "SouthWind.xml").getCanonicalPath()),
null);
// Question:
// Is this the best way to get the root element?
// get the root element
XdmNode rootNode = null;
for (XdmNode node : datasource.getXmlRootNode().children()) {
if (node.getNodeKind() == XdmNodeKind.ELEMENT) {
rootNode = node;
break;
}
}
TestBuildTree buildTree = new TestBuildTree(rootNode);
Element root = buildTree.addNode();
System.out.println("Schema:");
printElement("", root);
}
private static void printElement(String indent, Element element) {
System.out.println(indent + "<" + element.name + "> (" + (element.type == null ? "null" : element.type.getSimpleName()) + ")");
indent += " ";
for (Attribute attr : element.attributes)
System.out.println(indent + "=" + attr.name + " (" + (attr.type == null ? "null" : attr.type.getSimpleName()) + ")");
for (Element child : element.children)
printElement(indent, child);
}
protected XdmItem currentNode;
public TestBuildTree(XdmItem currentNode) {
this.currentNode = currentNode;
}
private Element addNode() throws SaxonApiException {
String name = ((XdmNode)currentNode).getNodeName().getLocalName();
// Question:
// Is this the best way to determine that this element has data (as opposed to child elements)?
Boolean elementHasData;
try {
((XdmNode) currentNode).getTypedValue();
elementHasData = true;
} catch (Exception ex) {
elementHasData = false;
}
// Questions:
// Is this the best way to get the type of the element value?
// If no schema is it always String?
Class valueClass = ! elementHasData ? null : ((XdmAtomicValue)((XdmNode)currentNode).getTypedValue()).getValue().getClass();
Element element = new Element(name, valueClass, null);
// add in attributes
XdmSequenceIterator currentSequence;
if ((currentSequence = moveTo(Axis.ATTRIBUTE)) != null) {
do {
name = ((XdmNode) currentNode).getNodeName().getLocalName();
// Questions:
// Is this the best way to get the type of the attribute value?
// If no schema is it always String?
valueClass = ((XdmAtomicValue)((XdmNode)currentNode).getTypedValue()).getValue().getClass();
Attribute attr = new Attribute(name, valueClass, null);
element.attributes.add(attr);
} while (moveToNextInCurrentSequence(currentSequence));
moveTo(Axis.PARENT);
}
// add in children elements
if ((currentSequence = moveTo(Axis.CHILD)) != null) {
do {
Element child = addNode();
// if we don't have this, add it
Element existing = element.getChildByName(child.name);
if (existing == null)
element.children.add(child);
else
// add in any children this does not have
existing.addNewItems (child);
} while (moveToNextInCurrentSequence(currentSequence));
moveTo(Axis.PARENT);
}
return element;
}
// moves to element or attribute
private XdmSequenceIterator moveTo(Axis axis) {
XdmSequenceIterator en = ((XdmNode) currentNode).axisIterator(axis);
boolean gotIt = false;
while (en.hasNext()) {
currentNode = en.next();
if (((XdmNode) currentNode).getNodeKind() == XdmNodeKind.ELEMENT || ((XdmNode) currentNode).getNodeKind() == XdmNodeKind.ATTRIBUTE) {
gotIt = true;
break;
}
}
if (gotIt) {
if (axis == Axis.ATTRIBUTE || axis == Axis.CHILD || axis == Axis.NAMESPACE)
return en;
return null;
}
return null;
}
// moves to next element or attribute
private Boolean moveToNextInCurrentSequence(XdmSequenceIterator currentSequence)
{
if (currentSequence == null)
return false;
while (currentSequence.hasNext())
{
currentNode = currentSequence.next();
if (((XdmNode)currentNode).getNodeKind() == XdmNodeKind.ELEMENT || ((XdmNode)currentNode).getNodeKind() == XdmNodeKind.ATTRIBUTE)
return true;
}
return false;
}
static class Node {
String name;
Class type;
String description;
public Node(String name, Class type, String description) {
this.name = name;
this.type = type;
this.description = description;
}
}
static class Element extends Node {
List<Element> children;
List<Attribute> attributes;
public Element(String name, Class type, String description) {
super(name, type, description);
children = new ArrayList<>();
attributes = new ArrayList<>();
}
public Element getChildByName(String name) {
for (Element child : children) {
if (child.name.equals(name))
return child;
}
return null;
}
public void addNewItems(Element child) {
for (Attribute attrAdd : child.attributes) {
boolean haveIt = false;
for (Attribute attrExist : attributes)
if (attrExist.name.equals(attrAdd.name)) {
haveIt = true;
break;
}
if (!haveIt)
attributes.add(attrAdd);
}
for (Element elemAdd : child.children) {
Element exist = null;
for (Element elemExist : children)
if (elemExist.name.equals(elemAdd.name)) {
exist = elemExist;
break;
}
if (exist == null)
children.add(elemAdd);
else
exist.addNewItems(elemAdd);
}
}
}
static class Attribute extends Node {
public Attribute(String name, Class type, String description) {
super(name, type, description);
}
}
}
XmlDatasource.java
import com.saxonica.config.EnterpriseConfiguration;
import com.saxonica.ee.s9api.SchemaValidatorImpl;
import net.sf.saxon.Configuration;
import net.sf.saxon.lib.FeatureKeys;
import net.sf.saxon.s9api.*;
import net.sf.saxon.type.SchemaException;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.XMLReaderFactory;
import javax.xml.transform.Source;
import javax.xml.transform.sax.SAXSource;
import javax.xml.transform.stream.StreamSource;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
public class XmlDatasource {
/** the DOM all searches are against */
private XdmNode xmlRootNode;
private XPathCompiler xPathCompiler;
/** key == the prefix; value == the uri mapped to that prefix */
private HashMap<String, String> prefixToUriMap = new HashMap<>();
/** key == the uri mapped to that prefix; value == the prefix */
private HashMap<String, String> uriToPrefixMap = new HashMap<>();
public XmlDatasource (InputStream xmlData, InputStream schemaFile) throws SAXException, SchemaException, SaxonApiException, IOException {
boolean haveSchema = schemaFile != null;
// call this before any instantiation of Saxon classes.
Configuration config = createEnterpriseConfiguration();
if (haveSchema) {
Source schemaSource = new StreamSource(schemaFile);
config.addSchemaSource(schemaSource);
}
Processor processor = new Processor(config);
DocumentBuilder doc_builder = processor.newDocumentBuilder();
XMLReader reader = createXMLReader();
InputSource xmlSource = new InputSource(xmlData);
SAXSource saxSource = new SAXSource(reader, xmlSource);
if (haveSchema) {
SchemaValidator validator = new SchemaValidatorImpl(processor);
doc_builder.setSchemaValidator(validator);
}
xmlRootNode = doc_builder.build(saxSource);
xPathCompiler = processor.newXPathCompiler();
if (haveSchema)
xPathCompiler.setSchemaAware(true);
declareNameSpaces();
}
public XdmNode getXmlRootNode() {
return xmlRootNode;
}
public XPathCompiler getxPathCompiler() {
return xPathCompiler;
}
/**
* Create a XMLReader set to disallow XXE aattacks.
* #return a safe XMLReader.
*/
public static XMLReader createXMLReader() throws SAXException {
XMLReader reader = XMLReaderFactory.createXMLReader();
// stop XXE https://www.owasp.org/index.php/XML_External_Entity_(XXE)_Prevention_Cheat_Sheet#JAXP_DocumentBuilderFactory.2C_SAXParserFactory_and_DOM4J
reader.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
reader.setFeature("http://xml.org/sax/features/external-general-entities", false);
reader.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
return reader;
}
private void declareNameSpaces() throws SaxonApiException {
// saxon has some of their functions set up with this.
prefixToUriMap.put("saxon", "http://saxon.sf.net");
uriToPrefixMap.put("http://saxon.sf.net", "saxon");
XdmValue list = xPathCompiler.evaluate("//namespace::*", xmlRootNode);
if (list == null || list.size() == 0)
return;
for (int index=0; index<list.size(); index++) {
XdmNode node = (XdmNode) list.itemAt(index);
String prefix = node.getNodeName() == null ? "" : node.getNodeName().getLocalName();
// xml, xsd, & xsi are XML structure ones, not ones used in the XML
if (prefix.equals("xml") || prefix.equals("xsd") || prefix.equals("xsi"))
continue;
// use default prefix if prefix is empty.
if (prefix == null || prefix.isEmpty())
prefix = "def";
// this returns repeats, so if a repeat, go on to next.
if (prefixToUriMap.containsKey(prefix))
continue;
String uri = node.getStringValue();
if (uri != null && !uri.isEmpty()) {
xPathCompiler.declareNamespace(prefix, uri);
prefixToUriMap.put(prefix, uri);
uriToPrefixMap.put(uri, prefix); }
}
}
public static EnterpriseConfiguration createEnterpriseConfiguration()
{
EnterpriseConfiguration configuration = new EnterpriseConfiguration();
configuration.supplyLicenseKey(new BufferedReader(new java.io.StringReader(deobfuscate(key))));
configuration.setConfigurationProperty(FeatureKeys.SUPPRESS_XPATH_WARNINGS, Boolean.TRUE);
return configuration;
}
}

Wrong step name displayed in Extent Report And replaced with When

I am using the given code in spec flow to generate the extent report. Test execute and report generated successfully but wrong steps name displayed in the extent report, And in the extent report replace with When. Steps come under And displayed in When while in the feature file steps written in And section.
Steps written under And syntax in feature files, displaying under When in Extent report.
I am using the following code to generate extent report.
using AventStack.ExtentReports;
using AventStack.ExtentReports.Gherkin.Model;
using AventStack.ExtentReports.Reporter;
using AventStack.ExtentReports.Reporter.Configuration;
using NUnit.Framework;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using TechTalk.SpecFlow;
namespace ALD_Belgium
{
[Binding]
[TestFixture]
class Hooks
{
public static ExtentTest featureName;
public static ExtentReports extent;
public static ExtentHtmlReporter htmlReporter;
public static ExtentTest test;
// public static object Theme { get; private set; }
static Hooks()
{
if (extent == null)
{
BasicSetUp();
}
}
[BeforeFeature]
public static void BeforeFeature()
{
featureName = extent.CreateTest<Feature>(FeatureContext.Current.FeatureInfo.Title);
}
[BeforeScenario]
public static void Setup()
{
BasePage.Intitialize();
BasePage.Navigate();
// test = extent.CreateTest(ScenarioContext.Current.ScenarioInfo.Title);
test = featureName.CreateNode<Scenario>(ScenarioContext.Current.ScenarioInfo.Title);
}
[AfterScenario]
public void TearDown()
{
if (ScenarioContext.Current.TestError != null)
{
var error = ScenarioContext.Current.TestError;
var errormessage = "<pre>" + error.Message + "</pre>";
extent.AddTestRunnerLogs(errormessage);
test.Log(Status.Error, errormessage);
test.Fail(errormessage);
}
BasePage.Quit();
}
[OneTimeSetUp]
public static void BasicSetUp()
{
string pth = System.Reflection.Assembly.GetCallingAssembly().CodeBase;
// string pth = System.IO.Directory.GetCurrentDirectory();
string actualPath = pth.Substring(0, pth.LastIndexOf("bin"));
string projectPath = new Uri(actualPath).LocalPath;
Console.WriteLine(" -----------Project Path--------------------------------------");
Console.WriteLine(projectPath);
string reportPath = projectPath + "Reports\\TestExecutionRunReport.html";
// Console.WriteLine("Report Path is " + reportPath);
htmlReporter = new ExtentHtmlReporter(reportPath);
htmlReporter.Configuration().Theme = Theme.Dark;
htmlReporter.Configuration().DocumentTitle = "SpecFlow Test Resport Document";
htmlReporter.Configuration().ReportName = "Feature Run Results";
extent = new ExtentReports();
extent.AttachReporter(htmlReporter);
//extent.LoadConfig(projectPath + "Extent-Config.xml");
}
[AfterTestRun]
public static void EndReport()
{
extent.Flush();
}
[AfterStep]
public static void InsertReportingSteps()
{
var stepType = ScenarioStepContext.Current.StepInfo.StepDefinitionType.ToString();
if (ScenarioContext.Current.TestError == null)
{
if (stepType == "Given")
test.CreateNode<Given>(ScenarioStepContext.Current.StepInfo.Text);
else if (stepType == "When")
test.CreateNode<When>(ScenarioStepContext.Current.StepInfo.Text);
else if (stepType == "And")
test.CreateNode<And>(ScenarioStepContext.Current.StepInfo.Text);
else if (stepType == "Then")
test.CreateNode<Then>(ScenarioStepContext.Current.StepInfo.Text);
}
else if (ScenarioContext.Current.TestError != null)
{
if (stepType == "Given")
test.CreateNode<Given>(ScenarioStepContext.Current.StepInfo.Text).Fail(ScenarioContext.Current.TestError.Message);
else if (stepType == "When")
test.CreateNode<When>(ScenarioStepContext.Current.StepInfo.Text).Fail(ScenarioContext.Current.TestError.Message);
else if (stepType == "And")
test.CreateNode<And>(ScenarioStepContext.Current.StepInfo.Text).Fail(ScenarioContext.Current.TestError.Message);
else if (stepType == "Then")
test.CreateNode<Then>(ScenarioStepContext.Current.StepInfo.Text).Fail(ScenarioContext.Current.TestError.Message);
}
}
}
}
enter image description here
The problem maybe because in SpecFlow main keywords are -
Given, When, Then
Can be that this report takes steps descriptions not from Features file but from code itself, where method, I think, is written with "When" keyword.

Avro (Microsoft) Serialization of derived type members missing

I am evaluating the performance of Microsoft's implementation of Avro, and at first I thought I was getting phenomenal performance until I realized it just wasn't serializing the entire message ;-)
In the following there is a simple hierarchy of messages decorated with [DataContract] (a base and two derived types). All members are decorated with the [DataMember] attribute. I create a serializer from the base message type and serialize a list of derived messages, but it appears to only serialize/deserialize the base class members. All of the derived message members are missing from the result.
Am I missing something? My application will require mixed message types.
FWIW I don't see any strings from the second derived type in the binary file, so I suspect the derived type members aren't being serialized.
Thanks, Greg
class Program
{
[DataContract(Name = "SideType", Namespace = "AvroMessage")]
public enum EventType
{
Unknown = 0,
One = 1,
Two = 2
}
[DataContract(Name = "MessageBase", Namespace = "AvroMessage")]
public class MessageBase
{
[DataMember(Name = "Subtype")]
public string Subtype;
[DataMember(Name = "Timestamp")]
public DateTime Timestamp;
[DataMember(Name = "GroupName")]
public string GroupName;
public override bool Equals(object obj)
{
MessageBase other = obj as MessageBase;
if (other == null) return false;
return Subtype == other.Subtype &&
Timestamp == other.Timestamp &&
GroupName == other.GroupName;
}
}
[DataContract(Name = "SubMessage1", Namespace = "AvroMessage")]
public class SubMessage1 : MessageBase
{
[DataMember(Name = "Volume")]
public int Volume;
[DataMember(Name = "Count")]
public int Count;
[DataMember(Name = "DetectedSide")]
public EventType Event;
public override bool Equals(object obj)
{
SubMessage1 other = obj as SubMessage1;
if (other == null) return false;
return Subtype == other.Subtype &&
Timestamp == other.Timestamp &&
GroupName == other.GroupName &&
Event == other.Event &&
Volume == other.Volume &&
Count == other.Count;
}
}
[DataContract(Name = "SubMessage2", Namespace = "AvroMessage")]
public class SubMessage2 : MessageBase
{
[DataMember(Name = "Name1")]
public string Name1;
[DataMember(Name = "Volume1")]
public int Volume1;
[DataMember(Name = "Name2")]
public string Name2;
[DataMember(Name = "Volume2")]
public int Volume2;
[DataMember(Name = "PriceMove")]
public double PriceMove;
public override bool Equals(object obj)
{
SubMessage2 other = obj as SubMessage2;
if (other == null) return false;
return Subtype == other.Subtype &&
Timestamp == other.Timestamp &&
GroupName == other.GroupName &&
Volume1 == other.Volume1 &&
Name1 == other.Name1 &&
Volume2 == other.Volume2 &&
Name2 == other.Name2 &&
PriceMove == other.PriceMove;
}
}
public class MessageFactory
{
public static IEnumerable<MessageBase> CreateMessages(int number)
{
Random ran = new Random();
List<MessageBase> retval = new List<MessageBase>();
for (int i = 0; i < number; i++)
{
if (ran.Next(2) == 0)
{
SubMessage1 sub1 = new SubMessage1();
sub1.Timestamp = DateTime.Now;
sub1.GroupName = "Group" + DateTime.Now.Millisecond.ToString();
sub1.Subtype = "SubMessag1";
sub1.Volume = ran.Next(10000);
sub1.Count = ran.Next(100);
if (ran.Next(2) == 0)
{
sub1.Event = EventType.One;
}
else
{
sub1.Event = EventType.Two;
}
retval.Add(sub1);
}
else
{
SubMessage2 sub2 = new SubMessage2();
sub2.Timestamp = DateTime.Now;
sub2.GroupName = "Group" + DateTime.Now.Millisecond.ToString();
sub2.Subtype = "SubMessag2";
sub2.Volume1 = ran.Next(1000);
sub2.PriceMove = ran.NextDouble() * 100 - 50;
sub2.Volume2 = ran.Next(1000);
sub2.Name1 = "Contract" + (DateTime.Now.Millisecond + ran.Next(5)).ToString();
sub2.Name2 = "Contract" + DateTime.Now.Millisecond.ToString();
retval.Add(sub2);
}
}
return retval;
}
}
public static void TestAvro(int count)
{
bool correct = false;
long serTicks = 0;
long deserTicks = 0;
Stopwatch sw = new Stopwatch();
sw.Reset();
var serializer = Microsoft.Hadoop.Avro.AvroSerializer.Create<MessageBase>();
MessageBase[] messages = new MessageBase[count];
using (var file = File.Create(#"C:\test_avro.bin"))
{
int i = 0;
foreach (var message in MessageFactory.CreateMessages(count))
{
messages[i++] = message;
sw.Start();
serializer.Serialize(file, message);
sw.Stop();
}
}
serTicks = sw.ElapsedTicks;
sw.Reset();
List<int> badMessages = new List<int>();
using (var file = File.OpenRead(#"C:\test_avro.bin"))
{
for (int i = 0; i < count; i++)
{
sw.Start();
MessageBase message = serializer.Deserialize(file);
sw.Stop();
SubMessage1 m1 = message as SubMessage1;
SubMessage2 m2 = message as SubMessage2;
bool areNull = (m1 == null) && (m2 == null); // Always true
if (!messages[i].Equals(message)) badMessages.Add(i);
}
}
deserTicks = sw.ElapsedTicks;
correct = badMessages.Count == 0;
long size = (new FileInfo(#"C:\test_proto.bin")).Length;
Console.WriteLine(String.Format("Correct: {0}, Time Out: {1}, , Time In: {2}, , Size: {3}", correct, serTicks, deserTicks, size));
}
static void Main(string[] args)
{
TestAvro(10000);
Console.ReadLine();
}
}
My bad - I forgot the KnownType attribute on the base class, one for each derived type. It works if you include the attributes.

Resources