[MOBY-guts] biomoby commit

Eddie Kawas kawas at dev.open-bio.org
Wed Feb 27 16:09:44 UTC 2008


kawas
Wed Feb 27 11:09:44 EST 2008
Update of /home/repository/moby/moby-live/Java/src/main/org/biomoby/client
In directory dev.open-bio.org:/tmp/cvs-serv10218/Java/src/main/org/biomoby/client

Modified Files:
	CentralDigestCachedImpl.java 
Log Message:
re-did the implementation of fillServiceTypeCache
* on empty cache, RDF is downloaded from the registry
* on updates, if the cache is stale to a certain degree, the RDF is downloded and parsed, otherwise multiple calls to central occur
* no extra files are created or stored when caching servicestypes
moby-live/Java/src/main/org/biomoby/client CentralDigestCachedImpl.java,1.29,1.30
===================================================================
RCS file: /home/repository/moby/moby-live/Java/src/main/org/biomoby/client/CentralDigestCachedImpl.java,v
retrieving revision 1.29
retrieving revision 1.30
diff -u -r1.29 -r1.30
--- /home/repository/moby/moby-live/Java/src/main/org/biomoby/client/CentralDigestCachedImpl.java	2008/02/27 14:59:11	1.29
+++ /home/repository/moby/moby-live/Java/src/main/org/biomoby/client/CentralDigestCachedImpl.java	2008/02/27 16:09:44	1.30
@@ -46,6 +46,7 @@
 import org.biomoby.shared.Utils;
 import org.biomoby.shared.extended.ServiceInstanceParser;
 import org.biomoby.shared.extended.DataTypeParser;
+import org.biomoby.shared.extended.ServiceTypeParser;
 import org.biomoby.shared.MobyPrimaryDataSet;
 import org.biomoby.shared.MobyPrimaryData;
 import org.biomoby.shared.MobyPrimaryDataSimple;
@@ -92,6 +93,7 @@
 
     private int datatype_threshold ;
     private int service_threshold ;
+    private int service_type_threshold ;
     
     /***************************************************************************
      * Create an instance that will access a default Moby registry and
@@ -117,9 +119,12 @@
 	datatype_threshold = Integer.getInteger("cache.threshold.datatypes", 7).intValue();
 	if (datatype_threshold < 0 || datatype_threshold > 100)
 	    datatype_threshold = 7;
-	service_threshold = Integer.getInteger("cache.threshold.services", 7).intValue();
+	service_threshold = Integer.getInteger("cache.threshold.services", 30).intValue();
 	if (service_threshold < 0 || service_threshold > 100)
 	    service_threshold = 30;
+	service_type_threshold = Integer.getInteger("cache.threshold.servicetypes", 5).intValue();
+	if (service_type_threshold < 0 || service_type_threshold > 100)
+	    service_type_threshold = 5;
 	
 
     }
@@ -589,192 +594,172 @@
      * the cache and remove them, or fetched missing ones if success add there
      * new LIST_FILE
      **************************************************************************/
-    @Override protected boolean fillServiceTypesCache()
+    protected boolean fillServiceTypesCache()
 	throws MobyException {
 	try {
-	    fireEvent(SERVICE_TYPES_START);
-	    // XML from API
+	    
+	    if (isCacheEmpty(serviceTypesCache)) {
+		fireEvent (SERVICE_TYPES_START);
+		// get a list file
+                String byAuthorityAsXML = getServiceTypesAsXML();
+                
+		// download RDF and fill it up
+		ServiceTypeParser sip = new ServiceTypeParser(getResourceURL(SERVICE_TYPES_RESOURCE_NAME));
+                MobyServiceType[] services = sip.getMobyServiceTypesFromRDF();
+                
+                fireEvent (SERVICE_TYPES_COUNT, new Integer (services.length));
+                for (MobyServiceType st : services) {
+                    fireEvent (SERVICE_TYPE_LOADING, st.getName());
+                    String s = createServiceTypeXML(st);
+                    store(serviceTypesCache, st.getName(), s);
+                    fireEvent (SERVICE_TYPE_LOADED, st.getName());
+                }
+                // process the root node
+                fireEvent (SERVICE_TYPE_LOADING, "Service");
+                String s = createServiceTypeXML(new MobyServiceType("Service"));
+                store(serviceTypesCache, "Service", s);
+                fireEvent (SERVICE_TYPE_LOADED, "Service");
+                
+                // finally, store the new LIST_FILE
+                store(serviceTypesCache, LIST_FILE, byAuthorityAsXML);
+                // done
+                return true;
+	    }
+	    
+	    fireEvent (SERVICE_TYPES_START);
 	    String typesAsXML = getServiceTypesAsXML();
 
 	    // get a list file with all service type names currently
 	    // in the cache...
 	    MobyServiceType[] cachedList = new MobyServiceType[] {};
-	    // XML from cache
-	    String xmlList = getListFile(serviceTypesCache);
+	    String xmlList = getListFile (serviceTypesCache);
 	    if (xmlList != null)
-		cachedList = createServiceTypesFromXML(xmlList);
+		cachedList = createServiceTypesFromXML (xmlList);
 
-	    // map of name -> service type
-	    HashMap<String, MobyServiceType> cachedTypes = new HashMap<String, MobyServiceType>();
+	    HashMap cachedTypes = new HashMap();
 	    for (int i = 0; i < cachedList.length; i++) {
-		cachedTypes.put(cachedList[i].getName(), cachedList[i]);
+		cachedTypes.put (cachedList[i].getName(), cachedList[i]);
 	    }
 
 	    // ...and remove it
-	    remove(serviceTypesCache, LIST_FILE);
+	    remove (serviceTypesCache, LIST_FILE);
 
 	    // get a list file with all service types from the
 	    // registry
-	    MobyServiceType[] types = createServiceTypesFromXML(typesAsXML);
-	    fireEvent(SERVICE_TYPES_COUNT, new Integer(types.length));
+	    MobyServiceType[] types = createServiceTypesFromXML (typesAsXML);
+	    fireEvent (SERVICE_TYPES_COUNT, new Integer (types.length));
+
+	    // list of current files in this cache
+	    HashSet currentFiles = new HashSet();
+	    File[] list = serviceTypesCache.listFiles();
+	    if (list == null)
+		throw new MobyException (MSG_CACHE_NOT_DIR (serviceTypesCache));
+	    for (int i = 0; i < list.length; i++) {
+		if (! ignored (list[i]))
+		    currentFiles.add (list[i].getName());
+	    }
+	    // a list of service types needed to fetch from the registry
+	    ArrayList<String> serviceTypesToFetch = new ArrayList<String>();
+	    // iterate over LIST_FILE and fetch missing files
+	    for (int i = 0 ; i < types.length; i++) {
+		boolean needToFetch = false;
+		String name = types[i].getName();
+		if ( ! currentFiles.contains (name)) {
+		    // missing file
+		    needToFetch = true;
+		} else {
+		    // check by comparing LSIDs
+		    String lsid = types[i].getLSID();
+		    if (cachedTypes.containsKey (name)) {
+			// should always go here - or we have a broken cache, anyway
+			String cachedLSID =
+			    ( (MobyServiceType)cachedTypes.get (name) ).getLSID();
+			if (! lsid.equals (cachedLSID)) {
+			    needToFetch = true;
+			}
+		    } else {
+			needToFetch = true;
+		    }
+		}
+		if (needToFetch) {
+		    serviceTypesToFetch.add(name);
+		}
+		currentFiles.remove (name);
+	    }
 
-	    boolean isStale = cachedTypes.size() != types.length;
-	    if (!isStale) {
-		for (MobyServiceType service : types) {
-		    String name = service.getName();
-		    // does the cache have the datatype?
-		    if (!cachedTypes.containsKey(name)) {
-			isStale = true;
-			break;
-		    }
-		    // are the lsids the same?
-		    MobyServiceType md = cachedTypes.remove(name);
-		    if (!md.getLSID().trim().equals(service.getLSID().trim())) {
-			isStale = true;
-			break;
-		    }
-		}
-		// we iterated over all of the types ... are there any in the
-		// cache that arent in the registry?
-		if (!isStale && cachedTypes.size() > 0) {
-		    isStale = true;
-		}
-	    }
-	    fireEvent(SERVICE_TYPE_LOADING, SERVICE_TYPES_RESOURCE_NAME
-		    + " RDF downloading");
-	    // make sure that the RDF file exists before we we try to read it
-	    // below
-	    try {
-		load(new File(serviceTypesCache, RDF_FILE));
-	    } catch (Exception e) {
-		isStale = true;
-	    }
-	    // if we are stale, fetch new RDF
-	    if (isStale) {
-		// store the RDF_FILE
-		String rdf = getResourceAsString(SERVICE_TYPES_RESOURCE_NAME);
-		store(serviceTypesCache, RDF_FILE, rdf.toString());
-	    }
-	    fireEvent(SERVICE_TYPE_LOADED, SERVICE_TYPES_RESOURCE_NAME
-		    + " RDF downloading");
-
-	    // load the RDF into memory
-	    fireEvent(SERVICE_TYPE_LOADING, SERVICE_TYPES_RESOURCE_NAME
-		    + " RDF parsing");
-	    MobyServiceType.loadServiceTypes(new URL("file:///"
-		    + serviceTypesCache.getAbsolutePath() + File.separator
-		    + RDF_FILE), reg);
-	    // set the flag that service types have been loaded
-	    fireEvent(SERVICE_TYPE_LOADED, SERVICE_TYPES_RESOURCE_NAME
-		    + " RDF parsing");
-	    setServiceTypesLoaded(true);
+	    if (((serviceTypesToFetch.size() *100)/types.length) > service_type_threshold) {
+		// download RDF and fill it up
+		ServiceTypeParser sip = new ServiceTypeParser(getResourceURL(SERVICE_TYPES_RESOURCE_NAME));
+                MobyServiceType[] services = sip.getMobyServiceTypesFromRDF();
+		fireEvent (SERVICE_TYPES_COUNT, new Integer (services.length));
+	        for (MobyServiceType st : services) {
+	            fireEvent(SERVICE_TYPE_LOADING, st.getName());
+		    // only process new ones
+	            if (serviceTypesToFetch.contains(st.getName())) {
+			String s = createServiceTypeXML(st);
+		    	store(serviceTypesCache, st.getName(), s);
+		    }
+		    fireEvent(SERVICE_TYPE_LOADED, st.getName());
+	        }
+		
+	    } else {
+		// use multiple api calls
+		for (String name : serviceTypesToFetch) {
+		    fireEvent (SERVICE_TYPE_LOADING, name);
+		    String xml = getServiceTypeRelationshipsAsXML(name, false);
+		    store(serviceTypesCache, name, xml);
+		    fireEvent(SERVICE_TYPE_LOADED, name);
+    		    if (stopST) {
+    			log.warn("Service types cache not fully updated");
+    			return false;
+    		    }
+		}
+	    }
+	    
+	    // remove files that are not any more needed
+	    for (Iterator it = currentFiles.iterator(); it.hasNext(); )
+		remove (serviceTypesCache, (String)it.next());
 
-	    // finally, store the new LIST_FILE
-	    store(serviceTypesCache, LIST_FILE, typesAsXML);
+	    // finally, put there the new LIST_FILE
+	    store (serviceTypesCache, LIST_FILE, typesAsXML);
 	    return true;
 
 	} catch (Exception e) {
-	    throw new MobyException(formatException(e), e);
+	    throw new MobyException (formatException (e), e);
 	} finally {
-	    fireEvent(stopST ? SERVICE_TYPES_CANCELLED : SERVICE_TYPES_END);
+	    fireEvent (stopST ? SERVICE_TYPES_CANCELLED :SERVICE_TYPES_END);
 	    stopST = false;
 	}
+}
+
+    private String createServiceTypeXML(MobyServiceType st) {
+	return "<Relationships>\n" + 
+		(st.getName().equals("Service") ? 
+			"" 
+			: 
+			"<Relationship relationshipType=\'ISA\' lsid=\'urn:lsid:biomoby.org:servicerelation:isa\'>\n" + 
+			"<serviceType lsid=\'"+st.getLSID()+"\' >"+ st.getParentName() +"</serviceType>\n" + 
+			"</Relationship>\n" ) + 
+		"</Relationships>\n";
     }
 
     /***************************************************************************
      * Update namespaces from a moby registry - this is easier than with other
      * entities: just get a new LIST_FILE.
      **************************************************************************/
-    @Override protected boolean fillNamespacesCache()
+    protected boolean fillNamespacesCache()
 	throws MobyException {
 	try {
-	    fireEvent(NAMESPACES_START);
-	    // XML from API
-	    String typesAsXML = getNamespacesAsXML();
-
-	    // get a list file with all namespaces names currently
-	    // in the cache...
-	    MobyNamespace[] cachedList = new MobyNamespace[] {};
-	    // XML from cache
-	    String xmlList = getListFile(namespacesCache);
-	    if (xmlList != null)
-		cachedList = createNamespacesFromXML(xmlList);
-
-	    // map of name -> service type
-	    HashMap<String, MobyNamespace> cachedTypes = new HashMap<String, MobyNamespace>();
-	    for (int i = 0; i < cachedList.length; i++) {
-		cachedTypes.put(cachedList[i].getName(), cachedList[i]);
-	    }
-
-	    // ...and remove it
-	    remove(namespacesCache, LIST_FILE);
-
-	    // get a list file with all namespaces from the
-	    // registry
-	    MobyNamespace[] types = createNamespacesFromXML(typesAsXML);
-	    fireEvent(NAMESPACES_COUNT, new Integer(types.length));
-
-	    boolean isStale = cachedTypes.size() != types.length;
-	    if (!isStale) {
-		for (MobyNamespace namespace : types) {
-		    String name = namespace.getName();
-		    // does the cache have the datatype?
-		    if (!cachedTypes.containsKey(name)) {
-			isStale = true;
-			break;
-		    }
-		    // are the lsids the same?
-		    MobyNamespace md = cachedTypes.remove(name);
-		    if (!md.getLSID().trim().equals(namespace.getLSID().trim())) {
-			isStale = true;
-			break;
-		    }
-		}
-		// we iterated over all of the types ... are there any in the
-		// cache that arent in the registry?
-		if (!isStale && cachedTypes.size() > 0) {
-		    isStale = true;
-		}
-	    }
-	    fireEvent(NAMESPACE_LOADING, NAMESPACES_RESOURCE_NAME
-		    + " RDF downloading");
-	    // make sure that the RDF file exists before we we try to read it
-	    // below
-	    try {
-		load(new File(namespacesCache, RDF_FILE));
-	    } catch (Exception e) {
-		isStale = true;
-	    }
-	    // if we are stale, fetch new RDF
-	    if (isStale) {
-		// store the RDF_FILE
-		String rdf = getResourceAsString(NAMESPACES_RESOURCE_NAME);
-		store(namespacesCache, RDF_FILE, rdf.toString());
-	    }
-	    fireEvent(NAMESPACE_LOADED, NAMESPACES_RESOURCE_NAME
-		    + " RDF downloading");
-
-	    // load the RDF into memory
-	    fireEvent(NAMESPACE_LOADING, NAMESPACES_RESOURCE_NAME
-		    + " RDF parsing");
-	    MobyNamespace.loadNamespaces(new URL("file:///"
-		    + namespacesCache.getAbsolutePath() + File.separator
-		    + RDF_FILE), reg);
-	    // set the flag that service types have been loaded
-	    fireEvent(NAMESPACE_LOADED, NAMESPACES_RESOURCE_NAME
-		    + " RDF parsing");
-	    setNamespacesLoaded(true);
-
-	    // finally, store the new LIST_FILE
-	    store(namespacesCache, LIST_FILE, typesAsXML);
+	    fireEvent (NAMESPACES_START);
+	    String xml = getNamespacesAsXML();
+	    store (namespacesCache, LIST_FILE, xml);
 	    return true;
-
 	} catch (Exception e) {
-	    throw new MobyException(formatException(e), e);
+	    throw new MobyException (formatException (e), e);
 	} finally {
-	    fireEvent(NAMESPACES_END);
+	    fireEvent (NAMESPACES_END);
 	}
-    }
+}
     
     /*************************************************************************
     *
@@ -912,98 +897,71 @@
     /***************************************************************************
      * 
      **************************************************************************/
-    public MobyNamespace[] getFullNamespaces() throws MobyException {
+    public MobyNamespace[] getFullNamespaces()
+	throws MobyException {
 	if (namespacesCache == null)
 	    return super.getFullNamespaces();
 	synchronized (namespacesCache) {
-	    if (isCacheEmpty(namespacesCache)) {
+	    if (isCacheEmpty (namespacesCache)) {
 		initCache();
 		fillNamespacesCache();
 	    }
 
 	    // get a list file (with all namespaces)
-	    String xmlList = getListFile(namespacesCache);
-	    if (xmlList == null || !rdfExists(namespacesCache)) {
+	    String xmlList = getListFile (namespacesCache);
+	    if (xmlList == null) {
 		initCache();
 		fillNamespacesCache();
-		xmlList = getListFile(namespacesCache);
+		xmlList = getListFile (namespacesCache);
 		if (xmlList == null)
 		    return new MobyNamespace[] {};
 	    }
-	    
-	    try {
-        	    if (!isNamespacesLoaded()) {
-        		MobyNamespace.loadNamespaces(new URL("file:///"
-        			    + namespacesCache.getAbsolutePath() + File.separator
-        			    + RDF_FILE), reg);
-        		setNamespacesLoaded(true);
-        	    }
-	    } catch (Exception e) {
-		    throw new MobyException(formatException(e), e);
-	    }
-	    
-	    String[] names = extractNamespacesFromXML(xmlList);
-	    ArrayList<MobyNamespace> types = new ArrayList<MobyNamespace>();
-	    for (String name : names) {
-		MobyNamespace namespace = MobyNamespace.getNamespace(name, reg);
-		if (namespace != null)
-		    types.add(namespace);
-		// else System.err.println(name + " is null for
-		// getNamespace()");
-	    }
-	    return types.toArray(new MobyNamespace[types.size()]);
+	    return createNamespacesFromXML (xmlList);
 	}
-    }
+}
 
     /***************************************************************************
      * 
      **************************************************************************/
-    protected MobyServiceType[] readServiceTypes() throws MobyException {
+    protected MobyServiceType[] readServiceTypes()
+	throws MobyException {
 	if (serviceTypesCache == null)
 	    return super.readServiceTypes();
 	synchronized (serviceTypesCache) {
-	    if (isCacheEmpty(serviceTypesCache)) {
+	    if (isCacheEmpty (serviceTypesCache)) {
 		initCache();
-		if (!fillServiceTypesCache())
+		if (! fillServiceTypesCache())
 		    // a callback stopped filling
 		    return new MobyServiceType[] {};
 	    }
 
 	    // get a list file (with all service type names)
-	    String xmlList = getListFile(serviceTypesCache);
-	    if (xmlList == null || !rdfExists(serviceTypesCache)) {
-		if (!fillServiceTypesCache())
+	    String xmlList = getListFile (serviceTypesCache);
+	    if (xmlList == null) {
+		if (! fillServiceTypesCache())
 		    // a callback stopped filling
 		    return new MobyServiceType[] {};
 		else {
-		    xmlList = getListFile(serviceTypesCache);
+		    xmlList = getListFile (serviceTypesCache);
 		    if (xmlList == null)
 			return new MobyServiceType[] {};
 		}
 	    }
-	    try {
-    	    if (!isServiceTypesLoaded()) {
-    		MobyServiceType.loadServiceTypes(new URL("file:///"
-    			    + serviceTypesCache.getAbsolutePath() + File.separator
-    			    + RDF_FILE), reg);
-    		setServiceTypesLoaded(true);
-    	    }
-	    } catch (Exception e) {
-		    throw new MobyException(formatException(e), e);
-	    }
+	    MobyServiceType[] types = createServiceTypesFromXML (xmlList);
 
-	    ArrayList<MobyServiceType> list = new ArrayList<MobyServiceType>();
-	    MobyServiceType[] types = createServiceTypesFromXML(xmlList);
 	    // add details about relationship to get full service types
 	    for (int i = 0; i < types.length; i++) {
 		String name = types[i].getName();
-		MobyServiceType type = MobyServiceType.getServiceType(name, reg); 
-		if (type != null)
-		    list.add(type);
+		File file = new File (serviceTypesCache, name);
+		try {
+		    types[i].setParentNames (createServiceTypeRelationshipsFromXML (load (file)));
+		} catch (MobyException e) {
+		    log.error (MSG_CACHE_BAD_FILE (file, e));
+		}
 	    }
-	    return list.toArray(new MobyServiceType[]{});
+	    return types;
 	}
-    }
+}
 
     /***************************************************************************
      * parse list file for just the names of the Namespaces
@@ -1026,33 +984,11 @@
 	return results;
     }
 
-    protected static boolean rdfExists(File cache) {
-	File rdfFile = new File(cache, RDF_FILE);
-	return rdfFile.exists();
-    }
-
     /***************************************************************************
      * Some file (when a cache is being tested for emptyness) are ignored.
      **************************************************************************/
-    @Override protected static boolean ignoredForEmptiness (File file) {
+    protected static boolean ignoredForEmptiness (File file) {
 	String path = file.getPath();
-	return path.endsWith("~") || path.endsWith("__R__D__F__");
-    }
-
-    private boolean isNamespacesLoaded() {
-	return namespacesLoaded;
-    }
-
-    private void setNamespacesLoaded(boolean namespacesLoaded) {
-	this.namespacesLoaded = namespacesLoaded;
-    }
-
-    private boolean isServiceTypesLoaded() {
-	return serviceTypesLoaded;
+	return path.endsWith("~");
     }
-
-    private void setServiceTypesLoaded(boolean serviceTypesLoaded) {
-	this.serviceTypesLoaded = serviceTypesLoaded;
-    }
-
 }




More information about the MOBY-guts mailing list