[MOBY-guts] biomoby commit
Paul Gordon
gordonp at dev.open-bio.org
Fri Apr 16 17:33:26 UTC 2010
gordonp
Fri Apr 16 13:33:26 EDT 2010
Update of /home/repository/moby/moby-live/Java/src/main/ca/ucalgary/seahawk/util
In directory dev.open-bio.org:/tmp/cvs-serv15287/src/main/ca/ucalgary/seahawk/util
Modified Files:
DataFlowRecorder.java DataUtils.java FilterSearch.java
MobyDataObjectDeferred.java XPointerResolver.java
Added Files:
MobyDataObjectDeferredSAI.java
MobyDataObjectSetDeferredSAI.java
Log Message:
First fully functional version of 'for each' service invocation in Seahawk
moby-live/Java/src/main/ca/ucalgary/seahawk/util MobyDataObjectDeferredSAI.java,NONE,1.1 MobyDataObjectSetDeferredSAI.java,NONE,1.1 DataFlowRecorder.java,1.4,1.5 DataUtils.java,1.5,1.6 FilterSearch.java,1.5,1.6 MobyDataObjectDeferred.java,1.1,1.2 XPointerResolver.java,1.1,1.2
===================================================================
RCS file: /home/repository/moby/moby-live/Java/src/main/ca/ucalgary/seahawk/util/DataFlowRecorder.java,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -r1.4 -r1.5
--- /home/repository/moby/moby-live/Java/src/main/ca/ucalgary/seahawk/util/DataFlowRecorder.java 2010/04/10 00:40:17 1.4
+++ /home/repository/moby/moby-live/Java/src/main/ca/ucalgary/seahawk/util/DataFlowRecorder.java 2010/04/16 17:33:26 1.5
@@ -321,7 +321,7 @@
data = sampleData.getUserData().toString().split("\t");
//options: selection + filter + cond, selection + cond, or cond only
if(data.length == 14 || data.length == 8 || data.length == 6){
- System.err.println("Adding conditional for " + resultURLString);
+ // System.err.println("Adding conditional for " + resultURLString);
String conditionURL = data[data.length-6];
String conditionRegex = data[data.length-5];
XPathOption conditionXPath = new XPathOption(data[data.length-4], data[data.length-3]);
@@ -334,10 +334,10 @@
System.arraycopy(data, 0, conditionlessData, 0, data.length-6);
data = conditionlessData;
}
- else{
- System.err.println("Skipping conditional, only " + data.length +
- "members in provenance data for " + resultURLString);
- }
+// else{
+// System.err.println("Skipping conditional, only " + data.length +
+// "members in provenance data for " + resultURLString);
+// }
}
// true means treat as a collection if that's what the sample data is
@@ -447,7 +447,7 @@
// Remember that sd is the loop's original MobyDataInstance before it's been coerced into a MobyPrimaryData,
// which doesn't have XML modes, etc.
String inputKey = getInputKey(sd);
- System.err.println("Input key for " + processorName + " is " + inputKey);
+ // System.err.println("Input key for " + processorName + " is " + inputKey);
String[] mobifyingProcessorNameAndPorts = null;
if(input2Processor.containsKey(inputKey)){
@@ -930,7 +930,7 @@
// supertype of the output from the other service!
if(namespace != null){
MobyNamespace nsObj = MobyNamespace.getNamespace(namespace, getRegistryFromService(srcService));
- System.err.println("About to create namespace filter for "+ srcProcessor + " port " + portName + " when srcPort was " + srcPort);
+ // System.err.println("About to create namespace filter for "+ srcProcessor + " port " + portName + " when srcPort was " + srcPort);
return createNamespaceFilter(nsObj, srcProcessor, portName, processors, datalinks, doc);
}
else if(xrefNs != null){
@@ -1103,7 +1103,7 @@
throws Exception{
String decompKey = srcProcessor+"\n"+srcPort+"\n"+xpath;
- System.err.println("Decomp key is " + decompKey);
+ // System.err.println("Decomp key is " + decompKey);
// Has this decomp already been created in another branch?
if(decomp2Processor.containsKey(decompKey)){
return decomp2Processor.get(decompKey);
===================================================================
RCS file: /home/repository/moby/moby-live/Java/src/main/ca/ucalgary/seahawk/util/DataUtils.java,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -r1.5 -r1.6
--- /home/repository/moby/moby-live/Java/src/main/ca/ucalgary/seahawk/util/DataUtils.java 2010/04/14 22:02:04 1.5
+++ /home/repository/moby/moby-live/Java/src/main/ca/ucalgary/seahawk/util/DataUtils.java 2010/04/16 17:33:26 1.6
@@ -166,6 +166,7 @@
Document resultDom = docBuilder.parse(serviceResultURL.openStream());
Registry registry = getRegistry(resultDom);
+ //todo: check all jobs, not just a sample
MobyDataJob sampleJob = getInputSample(resultDom, registry);
// Trace the inputs that came from the current document
@@ -343,6 +344,13 @@
* @param responseDom the XML DOM for the answer from a Moby Web service called in Seahawk
*/
public static MobyDataJob getInputSample(Document responseDom, Registry registry) throws Exception{
+ return getInputSample(responseDom, null, registry);
+ }
+
+ /**
+ * Same as two-arg, but retrieves a specific job's input (or null if not a job name found in the doc).
+ */
+ public static MobyDataJob getInputSample(Document responseDom, String jobName, Registry registry) throws Exception{
Document inputDoc = getInputDoc(responseDom);
if(inputDoc == null){
return null;
@@ -360,11 +368,14 @@
throw new Exception("Could not parse Moby output message from the given DOM");
}
+ if(jobName != null){
+ return inputPayload.get(jobName);
+ }
// Pick a job that produced output, if given the choice of more that one
- for(String jobName: inputPayload.keySet()){
- if(outputPayload.containsKey(jobName) &&
- outputPayload.get(jobName).size() > 0){
- return inputPayload.get(jobName);
+ for(String jobID: inputPayload.keySet()){
+ if(outputPayload.containsKey(jobID) &&
+ outputPayload.get(jobID).size() > 0){
+ return inputPayload.get(jobID);
}
}
return (MobyDataJob) inputPayload.values().iterator().next();
@@ -460,9 +471,9 @@
/**
* Loads targetURL, minus the nodes specified by the xpath keys in filteredXPtrs (map values are not currently used).
*/
- public static void filterDoc(Document domDoc, Map<String,String> filteredXPtrs){
+ public static void filterNodes(Node rootNode, Map<String,String> filteredXPtrs){
- if(domDoc == null){
+ if(rootNode == null){
logger.log(Level.SEVERE, "Error: Could not get MOBY document as DOM (empty or malformed document?)");
return;
}
@@ -474,7 +485,7 @@
// wrong as they refer to ordinality)
Vector<Node> nodesToDelete = new Vector<Node>();
for(String xptr: filteredXPtrs.keySet()){
- nodesToDelete.add(XPointerResolver.getNodeFromXPointer(domDoc, xptr));
+ nodesToDelete.add(XPointerResolver.getNodeFromXPointer(rootNode, xptr));
}
// Delete
@@ -497,86 +508,54 @@
* @param filteredXPtrs a map of the xpointers to data that should NOT be deserialized if children of the targetURL XPointer
* @param docFilter the current filter applied to the document in targetURL
*/
- public static MobyDataInstance loadMobyDataFromXPointer(URL targetURL, Map<String,String> filteredXPtrs, FilterSearch docFilter){
-
- URL currentURL = null;
- try{currentURL = new URL(targetURL.toString().replaceAll("#"+targetURL.getRef(), ""));}
- catch(Exception e){
- logger.log(Level.SEVERE, "Couldn't extract referenceless URL from " + targetURL, e);
- }
-
- // Build the DOM
- Document domDoc = null;
- try{
- domDoc = docBuilder.parse(targetURL.openStream());
- } catch(org.xml.sax.SAXException saxe){
- logger.log(Level.SEVERE, "The document defining the MOBY data " +
- "could not be parsed", saxe);
- return null;
- } catch(java.io.IOException ioe){
- logger.log(Level.SEVERE, "The document defining the MOBY data " +
- " could not be read (from " + targetURL + ")", ioe);
- return null;
- }
- // Does an in-place edit of domDoc
- filterDoc(domDoc, filteredXPtrs);
-
+ public static MobyDataInstance loadMobyDataFromXPointer(URL targetURL, Map<String,String> filteredXPtrs,
+ FilterSearch docFilter){
+
+ URL currentURL = null;
+ try{currentURL = new URL(targetURL.toString().replaceAll("#"+targetURL.getRef(), ""));}
+ catch(Exception e){
+ logger.log(Level.SEVERE, "Couldn't extract referenceless URL from " + targetURL, e);
+ }
+
+ // Build the DOM
+ Document domDoc = null;
+ try{
+ domDoc = docBuilder.parse(targetURL.openStream());
+ } catch(org.xml.sax.SAXException saxe){
+ logger.log(Level.SEVERE, "The document defining the MOBY data " +
+ "could not be parsed", saxe);
+ return null;
+ } catch(java.io.IOException ioe){
+ logger.log(Level.SEVERE, "The document defining the MOBY data " +
+ " could not be read (from " + targetURL + ")", ioe);
+ return null;
+ }
+
+ return loadMobyDataFromXPointer(currentURL, domDoc, targetURL.getRef(), filteredXPtrs, docFilter);
+ }
+
+ public static MobyDataInstance loadMobyDataFromXPointer(URL docURL, Document domDoc, String targetXptr,
+ Map<String,String> filteredXPtrs, FilterSearch docFilter){
// Are we dealing with a simple object or a complex one?
MobyDataInstance mobyData = null;
// Find the DOM fragment corresponding to the MOBY ID anchor specified
// in the link URL by using an XPath statement on the source MOBY doc
- //System.err.println("Retrieving complex object from XPointer " + targetURL);
// A child Xpointer is of the form /1/2/1/1, specifying the DOM child
- // descent path from the root node to get to the target node. The
- // equivalent XPath is /*[1]/*[2]/*[1]/*[1]
- String childXPath = targetURL.getRef().replaceAll("/(\\d+)", "/*[$1]");
-
- // Build and run the XPath statement
- Element mobyObject = null;
- NodeList idSearchResult = null;
- Object xobject = null;
- try{
- xobject = xPathFactory.newXPath().evaluate(childXPath,
- domDoc,
- XPathConstants.NODESET);
-
- // Check the results
- if(xobject != null){
- idSearchResult = (NodeList) xobject;
- }
- else{
- logger.log(Level.WARNING,"Could not find Moby object in document " + currentURL +
- ", referred to by the reference ID in Moby link " + targetURL +
- " (document changed?)");
- return null;
- }
- }
- catch(Exception e){// Syntax error
- logger.log(Level.SEVERE, "Error: Could not search Moby data instance for XPath " +
- childXPath + ")", e);
- return null;
- }
+ // descent path from the root node to get to the target node.
+ Element mobyObject = (Element) XPointerResolver.getNodeFromXPointer(domDoc, targetXptr);
- if(idSearchResult.getLength() == 0){
- logger.log(Level.WARNING, "Error: Could not find Moby data instance with XPath (" + childXPath +
- " in " + targetURL);
- return null;
- }
- if(idSearchResult.getLength() > 1){
- logger.log(Level.WARNING, "Error: Moby data instance could not beresolved because " +
- "there are multiple elements with XPath " + childXPath + " in " +
- currentURL);
- return null;
- }
- if(!(idSearchResult.item(0) instanceof Element)){
- logger.log(Level.WARNING, "Error: Moby data instance with XPath " + childXPath + " in " +
- currentURL + " was not an element as required!");
- return null;
+ // Does an in-place edit of domDoc
+ // Check if the returned data was supposed to be filtered. If so, don't filter.
+ if(filteredXPtrs != null && !filteredXPtrs.isEmpty() && !filteredXPtrs.containsKey(targetXptr)){
+ // Otherwise filter, so child elements of the data to return reflect the current filter conditions
+ // (e.g. a collection may be filtered to just a subset based on namespace, so retrieving the collection should only
+ // return that subset of children).
+ domDoc = (Document) domDoc.cloneNode(true); // true == deep
+ filterNodes(domDoc, filteredXPtrs);
}
- mobyObject = (Element) idSearchResult.item(0);
-
+
// Create the Java MOBY API object based on the linked document DOM fragment
try{
MobyDataInstance mdi = MobyDataObject.createInstanceFromDOM(mobyObject, SeahawkOptions.getRegistry());
@@ -587,8 +566,8 @@
mobyData = mdi;
}
else{
- logger.log(Level.WARNING, "Error: Moby data instance retrieved with XPath " + childXPath + " in " +
- currentURL + " was not a primary MOBY input object as expected");
+ logger.log(Level.WARNING, "Error: Moby data instance retrieved with XPath " + targetXptr +
+ " was not a primary MOBY input object as expected");
return null;
}
@@ -600,8 +579,8 @@
// For workflow creation, etc. associate the potential input data with its source generalized XPath
// Also, if there is currently a filter on the doc, store it too as useful info for workflow creation.
- setUserData(mobyData, currentURL, elementInContextToNameBasedXPath(mobyObject),
- targetURL.getRef(), docFilter);
+ setUserData(mobyData, docURL, elementInContextToNameBasedXPath(mobyObject),
+ targetXptr, docFilter);
}
catch(MobyException mobye){ // Logic error
logger.log(Level.SEVERE, "Error: Could not construct Moby data instance from document fragment: ",
@@ -656,6 +635,7 @@
// It's a complex type. Assume the type is why we are picking it.
// TODO: In future we may want to check if a parent type is allowed in
// this slot, and generalize to that.
+ // Also, account for namespace if present?
xpath = "/"+elName + xpath;
}
if(!(currentElement.getParentNode() instanceof Element)){
@@ -784,7 +764,7 @@
/**
* Find out which registry the service that created the reponse is registered in.
*
- * @param responseURL the service response whose origin we should trace
+ * @param responseDoc the service response whose origin we should trace
*/
public static Registry getRegistry(Document responseDoc) throws Exception{
String endpointURL = getSeahawkAttrFromDoc(responseDoc, REGISTRY_ATTR);
@@ -1035,16 +1015,20 @@
}
if(!inversed && shouldFilter ||
inversed && !shouldFilter){
- xPtrsToFilter.put(currXPtr, "todo");
+ xPtrsToFilter.put(currXPtr, "whatever");
}
}
//todo: filter objects without the HAS field at all...
return filterableXPtrs;
}
- // We need to enumerate the possible peer-sets for the selected data item. Is the user interested in
- // items in the same namespace, same data type, or same article name?
- public static NodeList getPeerElements(Document doc, MobyDataInstance mobyData, String peerMode){
+
+ /**
+ * We need to enumerate the possible peer-sets for the selected data item. Is the user interested in
+ * items in the same namespace/data type, or same article name?
+ */
+ public static NodeList getPeerElements(Document doc, MobyDataInstance mobyData,
+ Map<String,String> xPtrsToFilter, String peerMode){
if(!(mobyData instanceof MobyPrimaryData)){
logger.log(Level.WARNING, "Tried to get peers of data that was not an instanceof MobyPrimaryData (was " +
mobyData.getClass().getName()+ ")");
@@ -1054,10 +1038,13 @@
NodeList peerElements = null;
String peerGroupXPath = null;
+ Node contextNode = null;
if(mobyData.getUserData() != null){
String[] userData = ((String) mobyData.getUserData()).split("\t");
// first arg is url#nameBasedXPath
peerGroupXPath = userData[0].split("#", 2)[1];
+ contextNode = XPointerResolver.getNodeFromXPointer(doc, userData[1]);
+ //System.err.println("Loaded context node " + userData[1] + " as " + contextNode);
}
else{
// Fallback is to get all nodes with the same name...
@@ -1079,7 +1066,7 @@
mobyData.setUserData(newUserData);
// keep from job article name down
- peerGroupXPath = peerGroupXPath.replaceFirst("^.*/(Collection|Simple)", "//*");
+ //peerGroupXPath = peerGroupXPath.replaceFirst("^.*?/(Collection|Simple)", "//*");
// we want to go by articleName too to catch the full semantics of the peer group definition
peerGroupXPath += xpathArticleCondition;
}
@@ -1090,17 +1077,58 @@
// For xpaths given above, MOBY namespace is problem in XPath evaluation.
// solution: eliminate Moby envelope path parts: full path doesn't resolve in xpath for unknown reasons
//.replaceAll("/(MOBY|mobyContent|mobyData|Collection|Simple)","/moby:$1"); //attempt 1
- //.replaceAll("/([a-zA-Z_0-9\\-]+)(?=/|$)", "/*[local-name() = '$1']") //attempt 2
- //.replaceAll("/([a-zA-Z_0-9\\-]+)\\[", "/*[local-name() = '$1' and "); //attempt 2 cont'd
+ peerGroupXPath = peerGroupXPath.replaceAll("/([a-zA-Z_0-9\\-]+)", "/d:$1");
try{
- peerElements = (NodeList) xPathFactory.newXPath().evaluate(peerGroupXPath,
- doc,
- XPathConstants.NODESET);
+ XPath xpath = xPathFactory.newXPath();
+ if(contextNode != null){
+ xpath.setNamespaceContext(new NamespaceContextImpl(contextNode, "d"));
+ }
+ peerElements = (NodeList) xpath.evaluate(peerGroupXPath,
+ doc,
+ XPathConstants.NODESET);
} catch(Exception e){
logger.log(Level.SEVERE, "Could not evaluate UserData XPath "+peerGroupXPath, e);
}
- System.err.println("Got " + peerElements.getLength() + " peers for " + peerGroupXPath);
+ //System.err.println("Got " + peerElements.getLength() + " peers for " + peerGroupXPath);
+
+ // Remove peers that are currently filtered. Seems to be quicker than cloning the doc,
+ // filtering the node, then running the xpath. Assumes you don't do much with peerElements
+ // afterwards, or you should apply the filter to remove errant children, etc.)
+ if(xPtrsToFilter != null && !xPtrsToFilter.isEmpty()){
+ MutableNodeList filterPassedPeers = new MutableNodeList();
+ for(int i = 0; i < peerElements.getLength(); i++){
+ boolean filtered = false;
+ String peerXPtr = getXPtr(peerElements.item(i));
+ // Check all ancestors to see if they are filtered
+ for(String ancestorXPtr = peerXPtr;
+ ancestorXPtr.length() > 0;
+ ancestorXPtr = ancestorXPtr.replaceFirst("/\\d+$", "")){
+ if(xPtrsToFilter.containsKey(ancestorXPtr)){
+ filtered = true;
+ break;
+ }
+ }
+ if(!filtered){
+ Element peerElement = (Element) peerElements.item(i);
+ // Now check if any child nodes need to be filtered
+ Map<String,String> childrenToDelete = new HashMap<String,String>();
+ for(String xptr: xPtrsToFilter.keySet()){
+ // is it a child of the peer about to be put in the "passed filter" list?
+ if(xptr.indexOf(peerXPtr+"/") == 0){
+ childrenToDelete.put(xptr.substring(peerXPtr.length()), "whatever");
+ }
+ }
+ if(!childrenToDelete.isEmpty()){
+ peerElement = (Element) peerElements.item(i).cloneNode(true); //true = deep copy
+ filterNodes(peerElement, childrenToDelete);
+ }
+ filterPassedPeers.add(peerElement);
+ }
+ }
+ peerElements = filterPassedPeers;
+ //System.err.println("Got " + peerElements.getLength() + " peers that passed the filter");
+ }
return peerElements;
}
===================================================================
RCS file: /home/repository/moby/moby-live/Java/src/main/ca/ucalgary/seahawk/util/FilterSearch.java,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -r1.5 -r1.6
--- /home/repository/moby/moby-live/Java/src/main/ca/ucalgary/seahawk/util/FilterSearch.java 2010/04/14 22:02:04 1.5
+++ /home/repository/moby/moby-live/Java/src/main/ca/ucalgary/seahawk/util/FilterSearch.java 2010/04/16 17:33:26 1.6
@@ -121,11 +121,17 @@
return xpathOptions;
}
+ public String toString(){
+ return filterRegex.toString()+"\t"+getSelectedXPath().getXPath()+"\t"+
+ getSelectedXPath().getDesc()+"\t"+caseSensitivity+"\t"+inverseSelection;
+ }
+
public boolean equals(Object o){
if(o == null || !(o instanceof FilterSearch)){
return false;
}
FilterSearch other = (FilterSearch) o;
+ //System.err.println("Comparing "+this+"\n\nand\n\n"+o);
return filterRegex.toString().equals(other.filterRegex.toString()) &&
getSelectedXPath().getXPath().equals(other.getSelectedXPath().getXPath()) &&
caseSensitivity == other.caseSensitivity &&
===================================================================
RCS file: /home/repository/moby/moby-live/Java/src/main/ca/ucalgary/seahawk/util/MobyDataObjectDeferred.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -r1.1 -r1.2
--- /home/repository/moby/moby-live/Java/src/main/ca/ucalgary/seahawk/util/MobyDataObjectDeferred.java 2010/04/14 23:48:27 1.1
+++ /home/repository/moby/moby-live/Java/src/main/ca/ucalgary/seahawk/util/MobyDataObjectDeferred.java 2010/04/16 17:33:26 1.2
@@ -15,8 +15,8 @@
super(MobyDataType.getDataType(MobyTags.MOBYOBJECT, SeahawkOptions.getRegistry()), ns, "any_id");
dataCreator = listener;
}
- public MobyDataObjectDeferred(MobyDataType dt, MobyPayloadRequestListener listener){
- super(dt);
+ public MobyDataObjectDeferred(MobyDataType dt, MobyPayloadRequestListener listener) throws Exception{
+ super(dt, new MobyNamespace("unknown"), "any_id");
dataCreator = listener;
}
public MobyDataObjectDeferred(MobyDataType dt, MobyNamespace ns, MobyPayloadRequestListener listener) throws Exception{
===================================================================
RCS file: /home/repository/moby/moby-live/Java/src/main/ca/ucalgary/seahawk/util/XPointerResolver.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -r1.1 -r1.2
--- /home/repository/moby/moby-live/Java/src/main/ca/ucalgary/seahawk/util/XPointerResolver.java 2010/04/11 20:15:28 1.1
+++ /home/repository/moby/moby-live/Java/src/main/ca/ucalgary/seahawk/util/XPointerResolver.java 2010/04/16 17:33:26 1.2
@@ -15,20 +15,24 @@
* Note, this currently only works for stuff like /1/1/2/3/1/67,
* not all XPointers, which includes functions, etc.
*/
- public static Node getNodeFromXPointer(Document d, String xptr){
- if(d == null || xptr == null || xptr.length() < 2 || !xptr.startsWith("/1")){
+ public static Node getNodeFromXPointer(Node rootNode, String xptr){
+ if(rootNode == null || xptr == null || xptr.length() < 2){
return null;
}
- Node currentNode = d.getDocumentElement();
+ Node currentNode = rootNode;
if(currentNode == null){
return null;
}
- if(xptr.equals("/1")){
- return currentNode;
- }
+ if(rootNode instanceof Document){
+ currentNode = ((Document) rootNode).getDocumentElement();
+ if(xptr.equals("/1")){
+ return currentNode;
+ }
- // Get rid of leading "/1/"
- String[] ordinal_steps = xptr.substring(3).split("/");
+ // Get rid of leading "/1/"
+ xptr = xptr.substring(3);
+ }
+ String[] ordinal_steps = xptr.split("/");
if(ordinal_steps.length == 0){
return currentNode;
}
More information about the MOBY-guts
mailing list