[MOBY-guts] biomoby commit
Paul Gordon
gordonp at dev.open-bio.org
Wed Apr 18 15:53:59 UTC 2007
gordonp
Wed Apr 18 11:53:58 EDT 2007
Update of /home/repository/moby/moby-live/Java/src/main/ca/ucalgary/services
In directory dev.open-bio.org:/tmp/cvs-serv12154/src/main/ca/ucalgary/services
Modified Files:
ACDService.java
Log Message:
Added support for binary data passthrough, better ACD parameter handling, and change from stdin to temp files for input due to EMBOSS inconsistency of implementing acceptance of stdin
moby-live/Java/src/main/ca/ucalgary/services ACDService.java,1.2,1.3
===================================================================
RCS file: /home/repository/moby/moby-live/Java/src/main/ca/ucalgary/services/ACDService.java,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -r1.2 -r1.3
--- /home/repository/moby/moby-live/Java/src/main/ca/ucalgary/services/ACDService.java 2007/04/12 01:02:29 1.2
+++ /home/repository/moby/moby-live/Java/src/main/ca/ucalgary/services/ACDService.java 2007/04/18 15:53:58 1.3
@@ -48,6 +48,7 @@
// Keep track of what MOBY input parameters are to be converted to what ACD types
private Map<String,String> acdTypes;
+ private Map<String,String> acdBasicTypes; //holds e.g. string, when acdType for same key is "nucleotide sequence"
// For launching command-line programs
private Runtime runtime;
Vector<MobyDataSecondaryInstance> fixedSecondaryData;
@@ -57,6 +58,7 @@
public void init(){
super.init();
acdTypes = new HashMap<String,String>();
+ acdBasicTypes = new HashMap<String,String>();
runtime = Runtime.getRuntime();
fixedSecondaryData = new Vector<MobyDataSecondaryInstance>();
}
@@ -146,8 +148,9 @@
// Transform the moby data to text, unless it's binary data, which will be passed as decoded bytes
// Now, for binary data, we have to ignore any fields other than the Base64 encoded one. Sorry!
String tempFileSuffix = ".txt";
- if(mobyInputTemplate.getDataType().inheritsFrom(binaryDataType)){
- inputDataBytes = org.apache.axis.encoding.Base64.decode(MobyDataBytes.ENCODED_MEMBER_NAME);
+ if(inputData instanceof MobyDataBytes){
+ //System.err.println("Passing binary data to service");
+ inputDataBytes = ((MobyDataBytes) inputData).getBytes();
tempFileSuffix = ".bin";
}
else{
@@ -161,11 +164,15 @@
}
// Create the required command-line flag for the parameter
- if(inputs.length == 1){
+
+ // Is it a primary param whose value must be passed in directly, as opposed to in a file?
+ String basicType = acdBasicTypes.get(paramName);
+ if(basicType != null &&
+ (basicType.equals("string") || basicType.equals("integer") ||
+ basicType.equals("float") || basicType.equals("boolean"))){
// If only one input, we can pass it via stdin
command.add("-"+paramName);
- command.add("stdin");
- stdin = inputDataBytes;
+ command.add(new String(inputDataBytes));
}
else{
// Otherwise we need to create temporary files to store the data
@@ -183,7 +190,10 @@
// User-selected Secondary input
for(MobyDataSecondaryInstance mobySecondary: request.getSecondaryData()){
- // Make that no funny business is going on with shell escape characters.
+ // ASSUMPTION: Don't need to make sure that no funny business is
+ // going on with shell escape characters in the secondary names and
+ // values (potential security problem), because we call the string
+ // array form of Runtime.exec() later, which does not invoke a shell.
command.add("-"+mobySecondary.getName());
command.add(mobySecondary.getObject().toString());
}
@@ -204,7 +214,7 @@
// Create the required command-line flag for the parameter
if(outputs.length == 1){
- // If only one input, we can pass it via stdin
+ // If only one output, we can grab it via stdout
command.add("-"+paramName);
command.add("stdout");
}
@@ -299,13 +309,18 @@
private String runProgram(MobyDataJob request, String[] command, File workingDir, final byte[] input) throws Exception{
// TODO: ensure $embossRootDirName/lib is in the LD_LIBRARY_PATH,
// what is the equivalent in Windows?
- Process process = runtime.exec(command,
- new String[]{"EMBOSS="+embossRootDirName,
- "EMBOSS_ACDROOT="+acdRootDirName},
- workingDir);
+ //for(String cmdpart: command){
+ // System.err.print(cmdpart+" ");
+ //}
+ //System.err.println("");
+ final Process process = runtime.exec(command,
+ new String[]{"EMBOSS="+embossRootDirName,
+ "EMBOSS_ACDROOT="+acdRootDirName},
+ workingDir);
final OutputStream stdin = process.getOutputStream();
final InputStream stderr = process.getErrorStream();
final InputStream stdout = process.getInputStream();
+ final StringBuffer output = new StringBuffer();
// Echo the program's stderr
new Thread(){
@@ -323,51 +338,71 @@
}
}.start();
- // Write the data to the command's stdin, one line at a time
+ // Write the data to the command's stdin, in one fell swoop
new Thread(){
public void run(){
try{
- final int SENDING_BLOCK_SIZE = 1024;
- for(int i = 0; i < input.length; i += SENDING_BLOCK_SIZE){
- // Last chunk
- if(i + SENDING_BLOCK_SIZE > input.length){
- stdin.write(input, i, input.length-i);
- break;
- }
- // Feed the data to the command-line process in chunks
- else{
- stdin.write(input, i, SENDING_BLOCK_SIZE);
- stdin.flush();
- }
- }
+ stdin.write(input);
stdin.flush();
stdin.close();
}
catch(Exception e){
- System.err.println("Caught exception while sending data to the command: "+e);
+ // Okay, so sometimes Java sits there waiting to write data
+ // even after the write call (due to buffering?), then throws
+ // an Exception because the stream was closed in the main thread
+ // below. How do we distinguish between genuine write issues and this?
+ // By checking if the job is finished already and exited normally and produced data.
+ // If so, don't bother reporting the error, otherwise report it
+ // because the program may be missing data!
+ try{
+ Thread.sleep(1000);}
+ catch(Exception te){
+ System.err.println("Warning: grace period for stdin close " +
+ "not met, thread was interrupted: " + te);
+ }
+ //grace period between stream close and process end
+ boolean completed = true;
+ int eVal = 0;
+ try{
+ eVal = process.exitValue();
+ } catch(IllegalThreadStateException itse){
+ completed = false;
+ }
+ // 3 conditions, from more most severe to least
+ if(!completed){
+ System.err.println("Caught exception while sending data to the command" +
+ "(while process is still running): "+e);
+ }
+ else if(eVal != 0){
+ System.err.println("Caught exception while sending data to the command" +
+ "(and the process had a non-zero return value): "+e);
+ }
+ else if(output.length() == 0){
+ System.err.println("Caught exception while sending data to the command " +
+ "(and the process output is blank): "+e);
+ // Not *necessarily* a real error
+ // (grace time may be exceeded), but maybe,
+ // so print it anyway
+ e.printStackTrace();
+ }
+ else{
+ return; //all okay, we only got here becuase of the java superfluous buffering issue
+ }
e.printStackTrace();
}
}
}.start();
- final StringBuffer output = new StringBuffer();
- // Read the program's output (in a thread, to avoid deadlocks)
- new Thread(){
- public void run(){
- try{
- byte[] data = new byte[1024];
- for(int c = stdout.read(data, 0, data.length); c != -1; c = stdout.read(data, 0, 1024)){
- output.append(new String(data, 0, c));
- }
- }
- catch(Exception e){
- System.err.println("Caught exception while receiving data from the command-line program: "+e);
- e.printStackTrace();
- }
+ try{
+ byte[] data = new byte[1024];
+ for(int c = stdout.read(data, 0, data.length); c != -1; c = stdout.read(data, 0, 1024)){
+ output.append(new String(data, 0, c));
+ }
+ }
+ catch(Exception e){
+ throw new Exception("Caught exception while receiving data from the command-line program: "+e);
+ }
- }
- }.start();
-
int exitVal = process.waitFor();
if(exitVal != 0){
addException(new MobyServiceException(MobyServiceException.WARNING,
@@ -658,7 +693,11 @@
ACDFile parsedACDData = new ACDFile(acdFile);
configureServiceFromACDApplication(service, parsedACDData.getApplicationSection());
- configureServiceFromACDInput(service, parsedACDData.getInputSection());
+ List<Map<String,String>> combinedInput = new ArrayList<Map<String,String>>(parsedACDData.getInputSection());
+ combinedInput.addAll(parsedACDData.getRequiredParamsSection());
+ configureServiceFromACDInput(service, combinedInput);
+ //configureServiceFromACDInput(service, parsedACDData.getInputSection());
+ //configureServiceFromACDInput(service, parsedACDData.getRequiredParamsSection());
configureServiceFromACDParams(service, parsedACDData.getAdditionalParamsSection());
if(useAdvancedParams){
configureServiceFromACDParams(service, parsedACDData.getAdvancedParamsSection());
@@ -699,10 +738,10 @@
protected void configureServiceFromACDInput(MobyService service,
List<Map<String,String>> spec) throws Exception{
MobyPrimaryData[] mobyInputTypes = service.getPrimaryInputs();
- Map<String,MobyPrimaryData> paramUsed = new HashMap<String,MobyPrimaryData>();
+ Map<String,MobyPrimaryData> paramUnused = new HashMap<String,MobyPrimaryData>();
for(MobyPrimaryData mobyPrimaryInput: mobyInputTypes){
- paramUsed.put(mobyPrimaryInput.getName(), mobyPrimaryInput);
+ paramUnused.put(mobyPrimaryInput.getName(), mobyPrimaryInput);
}
for(MobyData acdInput: specToMoby(spec)){
@@ -710,9 +749,9 @@
String acdInputName = acdInput.getName();
// Make sure it maps to a MOBY Input
- if(!paramUsed.containsKey(acdInputName)){
+ if(!paramUnused.containsKey(acdInputName)){
String declInputNames = "";
- for(String inputName: paramUsed.keySet()){
+ for(String inputName: paramUnused.keySet()){
declInputNames = inputName+" ";
}
throw new Exception("A required ACD input parameter ("+acdInputName+
@@ -720,25 +759,26 @@
"servlet configuration ( "+declInputNames+")");
}
- MobyPrimaryData mobyPrimaryInput = paramUsed.get(acdInputName);
+ MobyPrimaryData mobyPrimaryInput = paramUnused.get(acdInputName);
if(!textClient.canProduceTextTypeFromMoby(acdTypes.get(acdInputName), mobyPrimaryInput)){
throw new Exception("No XSLT rules exist that can produce the requested " +
"text type '" + acdTypes.get(acdInputName) +
- "' from the given MOBY object type (" +
+ "' (acd input parameter " + acdInputName +
+ ") from the given MOBY object type (" +
mobyPrimaryInput.getDataType().getName()+")");
}
// If it exists, make sure it's of the right type
if(acdInput instanceof MobyPrimaryDataSimple &&
- paramUsed.get(acdInputName) instanceof MobyPrimaryDataSet){
+ paramUnused.get(acdInputName) instanceof MobyPrimaryDataSet){
throw new Exception("The MOBY input parameter \""+acdInputName+
"\" is a Collection, but the ACD input parameter is " +
"a simple (expected a declaration like \""+acdInputName+":"+
""+
- paramUsed.get(acdInputName).getDataType().getName()+
+ paramUnused.get(acdInputName).getDataType().getName()+
"\")");
}
- paramUsed.remove(acdInputName);
+ paramUnused.remove(acdInputName);
}
// Secondary params read from the ACD spec always get included
else if(acdInput instanceof MobySecondaryData){
@@ -751,9 +791,9 @@
}
}
- if(!paramUsed.isEmpty()){
+ if(!paramUnused.isEmpty()){
String unmappedParams = "";
- for(String param: paramUsed.keySet()){
+ for(String param: paramUnused.keySet()){
unmappedParams += " "+param;
}
throw new Exception("There are superfluous MOBY input parameters specified that do " +
@@ -852,26 +892,27 @@
// Secondary Parameter types
String additional = block.get("additional");
String deFault = block.get("default");
- if(additional != null && additional.toUpperCase().equals("Y") ||
- deFault != null && deFault.toUpperCase().equals("N")){
+ //System.err.println("param: " + acdName + "/" + acdType +" " + additional +" ("+deFault+")");
+ if((additional != null && additional.toUpperCase().equals("Y")) ||
+ deFault != null){
if("boolean".equals(acdType) || "toggle".equals(acdType)){
MobySecondaryData bool = new MobySecondaryData(acdName);
bool.setDataType(MobySecondaryData.BOOLEAN_TYPE);
- bool.setDefaultValue(block.get("default"));
+ bool.setDefaultValue(deFault);
bool.setDescription(block.get("help"));
mobyDataType = bool;
}
else if("string".equals(acdType) || "range".equals(acdType)){
MobySecondaryData string = new MobySecondaryData(acdName);
string.setDataType(MobySecondaryData.STRING_TYPE);
- string.setDefaultValue(block.get("default"));
+ string.setDefaultValue(deFault);
string.setDescription(block.get("help"));
mobyDataType = string;
}
else if("list".equals(acdType)){
MobySecondaryData enumeration = new MobySecondaryData(acdName);
enumeration.setDataType(MobySecondaryData.STRING_TYPE);
- enumeration.setDefaultValue(block.get("default"));
+ enumeration.setDefaultValue(deFault);
enumeration.setDescription(block.get("information"));
processACDList(enumeration, block.get("values"),
block.get("delimiter"), block.get("codedelimiter"));
@@ -880,7 +921,7 @@
else if("integer".equals(acdType)){
MobySecondaryData integer = new MobySecondaryData(acdName);
integer.setDataType(MobySecondaryData.INTEGER_TYPE);
- integer.setDefaultValue(block.get("default"));
+ integer.setDefaultValue(deFault);
integer.setDescription(block.get("information"));
integer.setMinValue(block.get("minimum"));
integer.setMaxValue(block.get("maximum"));
@@ -889,7 +930,7 @@
else if("float".equals(acdType)){
MobySecondaryData floating = new MobySecondaryData(acdName);
floating.setDataType(MobySecondaryData.FLOAT_TYPE);
- floating.setDefaultValue(block.get("default"));
+ floating.setDefaultValue(deFault);
floating.setDescription(block.get("information"));
floating.setMinValue(block.get("minimum"));
floating.setMaxValue(block.get("maximum"));
@@ -897,6 +938,7 @@
}
else{
System.err.println("Skipping secondary parameter that is not a primitive in MOBY:" + acdType);
+ continue;
}
} //end if(additional)
// else it's a required parameter
@@ -910,10 +952,26 @@
else{
mobyDataType = new MobyPrimaryDataSimple(acdName);
}
+
+ // There are instances where the ACD data type is generic, such as
+ // "infile", "outfile", "string", and "datafile"
+ // We will not handle the case of "list" data type, because this was enumerated anyway.
+ if((acdType.equals("infile") ||
+ acdType.equals("outfile") ||
+ acdType.equals("string") ||
+ acdType.equals("datafile")) &&
+ block.containsKey("knowntype")){
+ // In such cases, keep track of the basic type (e.g. string, so when we build
+ // the command line we know how to represent the data)
+ acdBasicTypes.put(acdName, acdType);
+ acdType = block.get("knowntype");
+ }
acdTypes.put(acdName, acdType);
}
- mobyDataTypes.add(mobyDataType);
+ if(mobyDataType != null){
+ mobyDataTypes.add(mobyDataType);
+ }
}
return mobyDataTypes;
More information about the MOBY-guts
mailing list