[MOBY-guts] biomoby commit
Eddie Kawas
kawas at dev.open-bio.org
Wed Feb 27 16:09:44 UTC 2008
kawas
Wed Feb 27 11:09:44 EST 2008
Update of /home/repository/moby/moby-live/Java/src/main/org/biomoby/client
In directory dev.open-bio.org:/tmp/cvs-serv10218/Java/src/main/org/biomoby/client
Modified Files:
CentralDigestCachedImpl.java
Log Message:
re-did the implementation of fillServiceTypeCache
* on empty cache, RDF is downloaded from the registry
* on updates, if the cache is stale to a certain degree, the RDF is downloded and parsed, otherwise multiple calls to central occur
* no extra files are created or stored when caching servicestypes
moby-live/Java/src/main/org/biomoby/client CentralDigestCachedImpl.java,1.29,1.30
===================================================================
RCS file: /home/repository/moby/moby-live/Java/src/main/org/biomoby/client/CentralDigestCachedImpl.java,v
retrieving revision 1.29
retrieving revision 1.30
diff -u -r1.29 -r1.30
--- /home/repository/moby/moby-live/Java/src/main/org/biomoby/client/CentralDigestCachedImpl.java 2008/02/27 14:59:11 1.29
+++ /home/repository/moby/moby-live/Java/src/main/org/biomoby/client/CentralDigestCachedImpl.java 2008/02/27 16:09:44 1.30
@@ -46,6 +46,7 @@
import org.biomoby.shared.Utils;
import org.biomoby.shared.extended.ServiceInstanceParser;
import org.biomoby.shared.extended.DataTypeParser;
+import org.biomoby.shared.extended.ServiceTypeParser;
import org.biomoby.shared.MobyPrimaryDataSet;
import org.biomoby.shared.MobyPrimaryData;
import org.biomoby.shared.MobyPrimaryDataSimple;
@@ -92,6 +93,7 @@
private int datatype_threshold ;
private int service_threshold ;
+ private int service_type_threshold ;
/***************************************************************************
* Create an instance that will access a default Moby registry and
@@ -117,9 +119,12 @@
datatype_threshold = Integer.getInteger("cache.threshold.datatypes", 7).intValue();
if (datatype_threshold < 0 || datatype_threshold > 100)
datatype_threshold = 7;
- service_threshold = Integer.getInteger("cache.threshold.services", 7).intValue();
+ service_threshold = Integer.getInteger("cache.threshold.services", 30).intValue();
if (service_threshold < 0 || service_threshold > 100)
service_threshold = 30;
+ service_type_threshold = Integer.getInteger("cache.threshold.servicetypes", 5).intValue();
+ if (service_type_threshold < 0 || service_type_threshold > 100)
+ service_type_threshold = 5;
}
@@ -589,192 +594,172 @@
* the cache and remove them, or fetched missing ones if success add there
* new LIST_FILE
**************************************************************************/
- @Override protected boolean fillServiceTypesCache()
+ protected boolean fillServiceTypesCache()
throws MobyException {
try {
- fireEvent(SERVICE_TYPES_START);
- // XML from API
+
+ if (isCacheEmpty(serviceTypesCache)) {
+ fireEvent (SERVICE_TYPES_START);
+ // get a list file
+ String byAuthorityAsXML = getServiceTypesAsXML();
+
+ // download RDF and fill it up
+ ServiceTypeParser sip = new ServiceTypeParser(getResourceURL(SERVICE_TYPES_RESOURCE_NAME));
+ MobyServiceType[] services = sip.getMobyServiceTypesFromRDF();
+
+ fireEvent (SERVICE_TYPES_COUNT, new Integer (services.length));
+ for (MobyServiceType st : services) {
+ fireEvent (SERVICE_TYPE_LOADING, st.getName());
+ String s = createServiceTypeXML(st);
+ store(serviceTypesCache, st.getName(), s);
+ fireEvent (SERVICE_TYPE_LOADED, st.getName());
+ }
+ // process the root node
+ fireEvent (SERVICE_TYPE_LOADING, "Service");
+ String s = createServiceTypeXML(new MobyServiceType("Service"));
+ store(serviceTypesCache, "Service", s);
+ fireEvent (SERVICE_TYPE_LOADED, "Service");
+
+ // finally, store the new LIST_FILE
+ store(serviceTypesCache, LIST_FILE, byAuthorityAsXML);
+ // done
+ return true;
+ }
+
+ fireEvent (SERVICE_TYPES_START);
String typesAsXML = getServiceTypesAsXML();
// get a list file with all service type names currently
// in the cache...
MobyServiceType[] cachedList = new MobyServiceType[] {};
- // XML from cache
- String xmlList = getListFile(serviceTypesCache);
+ String xmlList = getListFile (serviceTypesCache);
if (xmlList != null)
- cachedList = createServiceTypesFromXML(xmlList);
+ cachedList = createServiceTypesFromXML (xmlList);
- // map of name -> service type
- HashMap<String, MobyServiceType> cachedTypes = new HashMap<String, MobyServiceType>();
+ HashMap cachedTypes = new HashMap();
for (int i = 0; i < cachedList.length; i++) {
- cachedTypes.put(cachedList[i].getName(), cachedList[i]);
+ cachedTypes.put (cachedList[i].getName(), cachedList[i]);
}
// ...and remove it
- remove(serviceTypesCache, LIST_FILE);
+ remove (serviceTypesCache, LIST_FILE);
// get a list file with all service types from the
// registry
- MobyServiceType[] types = createServiceTypesFromXML(typesAsXML);
- fireEvent(SERVICE_TYPES_COUNT, new Integer(types.length));
+ MobyServiceType[] types = createServiceTypesFromXML (typesAsXML);
+ fireEvent (SERVICE_TYPES_COUNT, new Integer (types.length));
+
+ // list of current files in this cache
+ HashSet currentFiles = new HashSet();
+ File[] list = serviceTypesCache.listFiles();
+ if (list == null)
+ throw new MobyException (MSG_CACHE_NOT_DIR (serviceTypesCache));
+ for (int i = 0; i < list.length; i++) {
+ if (! ignored (list[i]))
+ currentFiles.add (list[i].getName());
+ }
+ // a list of service types needed to fetch from the registry
+ ArrayList<String> serviceTypesToFetch = new ArrayList<String>();
+ // iterate over LIST_FILE and fetch missing files
+ for (int i = 0 ; i < types.length; i++) {
+ boolean needToFetch = false;
+ String name = types[i].getName();
+ if ( ! currentFiles.contains (name)) {
+ // missing file
+ needToFetch = true;
+ } else {
+ // check by comparing LSIDs
+ String lsid = types[i].getLSID();
+ if (cachedTypes.containsKey (name)) {
+ // should always go here - or we have a broken cache, anyway
+ String cachedLSID =
+ ( (MobyServiceType)cachedTypes.get (name) ).getLSID();
+ if (! lsid.equals (cachedLSID)) {
+ needToFetch = true;
+ }
+ } else {
+ needToFetch = true;
+ }
+ }
+ if (needToFetch) {
+ serviceTypesToFetch.add(name);
+ }
+ currentFiles.remove (name);
+ }
- boolean isStale = cachedTypes.size() != types.length;
- if (!isStale) {
- for (MobyServiceType service : types) {
- String name = service.getName();
- // does the cache have the datatype?
- if (!cachedTypes.containsKey(name)) {
- isStale = true;
- break;
- }
- // are the lsids the same?
- MobyServiceType md = cachedTypes.remove(name);
- if (!md.getLSID().trim().equals(service.getLSID().trim())) {
- isStale = true;
- break;
- }
- }
- // we iterated over all of the types ... are there any in the
- // cache that arent in the registry?
- if (!isStale && cachedTypes.size() > 0) {
- isStale = true;
- }
- }
- fireEvent(SERVICE_TYPE_LOADING, SERVICE_TYPES_RESOURCE_NAME
- + " RDF downloading");
- // make sure that the RDF file exists before we we try to read it
- // below
- try {
- load(new File(serviceTypesCache, RDF_FILE));
- } catch (Exception e) {
- isStale = true;
- }
- // if we are stale, fetch new RDF
- if (isStale) {
- // store the RDF_FILE
- String rdf = getResourceAsString(SERVICE_TYPES_RESOURCE_NAME);
- store(serviceTypesCache, RDF_FILE, rdf.toString());
- }
- fireEvent(SERVICE_TYPE_LOADED, SERVICE_TYPES_RESOURCE_NAME
- + " RDF downloading");
-
- // load the RDF into memory
- fireEvent(SERVICE_TYPE_LOADING, SERVICE_TYPES_RESOURCE_NAME
- + " RDF parsing");
- MobyServiceType.loadServiceTypes(new URL("file:///"
- + serviceTypesCache.getAbsolutePath() + File.separator
- + RDF_FILE), reg);
- // set the flag that service types have been loaded
- fireEvent(SERVICE_TYPE_LOADED, SERVICE_TYPES_RESOURCE_NAME
- + " RDF parsing");
- setServiceTypesLoaded(true);
+ if (((serviceTypesToFetch.size() *100)/types.length) > service_type_threshold) {
+ // download RDF and fill it up
+ ServiceTypeParser sip = new ServiceTypeParser(getResourceURL(SERVICE_TYPES_RESOURCE_NAME));
+ MobyServiceType[] services = sip.getMobyServiceTypesFromRDF();
+ fireEvent (SERVICE_TYPES_COUNT, new Integer (services.length));
+ for (MobyServiceType st : services) {
+ fireEvent(SERVICE_TYPE_LOADING, st.getName());
+ // only process new ones
+ if (serviceTypesToFetch.contains(st.getName())) {
+ String s = createServiceTypeXML(st);
+ store(serviceTypesCache, st.getName(), s);
+ }
+ fireEvent(SERVICE_TYPE_LOADED, st.getName());
+ }
+
+ } else {
+ // use multiple api calls
+ for (String name : serviceTypesToFetch) {
+ fireEvent (SERVICE_TYPE_LOADING, name);
+ String xml = getServiceTypeRelationshipsAsXML(name, false);
+ store(serviceTypesCache, name, xml);
+ fireEvent(SERVICE_TYPE_LOADED, name);
+ if (stopST) {
+ log.warn("Service types cache not fully updated");
+ return false;
+ }
+ }
+ }
+
+ // remove files that are not any more needed
+ for (Iterator it = currentFiles.iterator(); it.hasNext(); )
+ remove (serviceTypesCache, (String)it.next());
- // finally, store the new LIST_FILE
- store(serviceTypesCache, LIST_FILE, typesAsXML);
+ // finally, put there the new LIST_FILE
+ store (serviceTypesCache, LIST_FILE, typesAsXML);
return true;
} catch (Exception e) {
- throw new MobyException(formatException(e), e);
+ throw new MobyException (formatException (e), e);
} finally {
- fireEvent(stopST ? SERVICE_TYPES_CANCELLED : SERVICE_TYPES_END);
+ fireEvent (stopST ? SERVICE_TYPES_CANCELLED :SERVICE_TYPES_END);
stopST = false;
}
+}
+
+ private String createServiceTypeXML(MobyServiceType st) {
+ return "<Relationships>\n" +
+ (st.getName().equals("Service") ?
+ ""
+ :
+ "<Relationship relationshipType=\'ISA\' lsid=\'urn:lsid:biomoby.org:servicerelation:isa\'>\n" +
+ "<serviceType lsid=\'"+st.getLSID()+"\' >"+ st.getParentName() +"</serviceType>\n" +
+ "</Relationship>\n" ) +
+ "</Relationships>\n";
}
/***************************************************************************
* Update namespaces from a moby registry - this is easier than with other
* entities: just get a new LIST_FILE.
**************************************************************************/
- @Override protected boolean fillNamespacesCache()
+ protected boolean fillNamespacesCache()
throws MobyException {
try {
- fireEvent(NAMESPACES_START);
- // XML from API
- String typesAsXML = getNamespacesAsXML();
-
- // get a list file with all namespaces names currently
- // in the cache...
- MobyNamespace[] cachedList = new MobyNamespace[] {};
- // XML from cache
- String xmlList = getListFile(namespacesCache);
- if (xmlList != null)
- cachedList = createNamespacesFromXML(xmlList);
-
- // map of name -> service type
- HashMap<String, MobyNamespace> cachedTypes = new HashMap<String, MobyNamespace>();
- for (int i = 0; i < cachedList.length; i++) {
- cachedTypes.put(cachedList[i].getName(), cachedList[i]);
- }
-
- // ...and remove it
- remove(namespacesCache, LIST_FILE);
-
- // get a list file with all namespaces from the
- // registry
- MobyNamespace[] types = createNamespacesFromXML(typesAsXML);
- fireEvent(NAMESPACES_COUNT, new Integer(types.length));
-
- boolean isStale = cachedTypes.size() != types.length;
- if (!isStale) {
- for (MobyNamespace namespace : types) {
- String name = namespace.getName();
- // does the cache have the datatype?
- if (!cachedTypes.containsKey(name)) {
- isStale = true;
- break;
- }
- // are the lsids the same?
- MobyNamespace md = cachedTypes.remove(name);
- if (!md.getLSID().trim().equals(namespace.getLSID().trim())) {
- isStale = true;
- break;
- }
- }
- // we iterated over all of the types ... are there any in the
- // cache that arent in the registry?
- if (!isStale && cachedTypes.size() > 0) {
- isStale = true;
- }
- }
- fireEvent(NAMESPACE_LOADING, NAMESPACES_RESOURCE_NAME
- + " RDF downloading");
- // make sure that the RDF file exists before we we try to read it
- // below
- try {
- load(new File(namespacesCache, RDF_FILE));
- } catch (Exception e) {
- isStale = true;
- }
- // if we are stale, fetch new RDF
- if (isStale) {
- // store the RDF_FILE
- String rdf = getResourceAsString(NAMESPACES_RESOURCE_NAME);
- store(namespacesCache, RDF_FILE, rdf.toString());
- }
- fireEvent(NAMESPACE_LOADED, NAMESPACES_RESOURCE_NAME
- + " RDF downloading");
-
- // load the RDF into memory
- fireEvent(NAMESPACE_LOADING, NAMESPACES_RESOURCE_NAME
- + " RDF parsing");
- MobyNamespace.loadNamespaces(new URL("file:///"
- + namespacesCache.getAbsolutePath() + File.separator
- + RDF_FILE), reg);
- // set the flag that service types have been loaded
- fireEvent(NAMESPACE_LOADED, NAMESPACES_RESOURCE_NAME
- + " RDF parsing");
- setNamespacesLoaded(true);
-
- // finally, store the new LIST_FILE
- store(namespacesCache, LIST_FILE, typesAsXML);
+ fireEvent (NAMESPACES_START);
+ String xml = getNamespacesAsXML();
+ store (namespacesCache, LIST_FILE, xml);
return true;
-
} catch (Exception e) {
- throw new MobyException(formatException(e), e);
+ throw new MobyException (formatException (e), e);
} finally {
- fireEvent(NAMESPACES_END);
+ fireEvent (NAMESPACES_END);
}
- }
+}
/*************************************************************************
*
@@ -912,98 +897,71 @@
/***************************************************************************
*
**************************************************************************/
- public MobyNamespace[] getFullNamespaces() throws MobyException {
+ public MobyNamespace[] getFullNamespaces()
+ throws MobyException {
if (namespacesCache == null)
return super.getFullNamespaces();
synchronized (namespacesCache) {
- if (isCacheEmpty(namespacesCache)) {
+ if (isCacheEmpty (namespacesCache)) {
initCache();
fillNamespacesCache();
}
// get a list file (with all namespaces)
- String xmlList = getListFile(namespacesCache);
- if (xmlList == null || !rdfExists(namespacesCache)) {
+ String xmlList = getListFile (namespacesCache);
+ if (xmlList == null) {
initCache();
fillNamespacesCache();
- xmlList = getListFile(namespacesCache);
+ xmlList = getListFile (namespacesCache);
if (xmlList == null)
return new MobyNamespace[] {};
}
-
- try {
- if (!isNamespacesLoaded()) {
- MobyNamespace.loadNamespaces(new URL("file:///"
- + namespacesCache.getAbsolutePath() + File.separator
- + RDF_FILE), reg);
- setNamespacesLoaded(true);
- }
- } catch (Exception e) {
- throw new MobyException(formatException(e), e);
- }
-
- String[] names = extractNamespacesFromXML(xmlList);
- ArrayList<MobyNamespace> types = new ArrayList<MobyNamespace>();
- for (String name : names) {
- MobyNamespace namespace = MobyNamespace.getNamespace(name, reg);
- if (namespace != null)
- types.add(namespace);
- // else System.err.println(name + " is null for
- // getNamespace()");
- }
- return types.toArray(new MobyNamespace[types.size()]);
+ return createNamespacesFromXML (xmlList);
}
- }
+}
/***************************************************************************
*
**************************************************************************/
- protected MobyServiceType[] readServiceTypes() throws MobyException {
+ protected MobyServiceType[] readServiceTypes()
+ throws MobyException {
if (serviceTypesCache == null)
return super.readServiceTypes();
synchronized (serviceTypesCache) {
- if (isCacheEmpty(serviceTypesCache)) {
+ if (isCacheEmpty (serviceTypesCache)) {
initCache();
- if (!fillServiceTypesCache())
+ if (! fillServiceTypesCache())
// a callback stopped filling
return new MobyServiceType[] {};
}
// get a list file (with all service type names)
- String xmlList = getListFile(serviceTypesCache);
- if (xmlList == null || !rdfExists(serviceTypesCache)) {
- if (!fillServiceTypesCache())
+ String xmlList = getListFile (serviceTypesCache);
+ if (xmlList == null) {
+ if (! fillServiceTypesCache())
// a callback stopped filling
return new MobyServiceType[] {};
else {
- xmlList = getListFile(serviceTypesCache);
+ xmlList = getListFile (serviceTypesCache);
if (xmlList == null)
return new MobyServiceType[] {};
}
}
- try {
- if (!isServiceTypesLoaded()) {
- MobyServiceType.loadServiceTypes(new URL("file:///"
- + serviceTypesCache.getAbsolutePath() + File.separator
- + RDF_FILE), reg);
- setServiceTypesLoaded(true);
- }
- } catch (Exception e) {
- throw new MobyException(formatException(e), e);
- }
+ MobyServiceType[] types = createServiceTypesFromXML (xmlList);
- ArrayList<MobyServiceType> list = new ArrayList<MobyServiceType>();
- MobyServiceType[] types = createServiceTypesFromXML(xmlList);
// add details about relationship to get full service types
for (int i = 0; i < types.length; i++) {
String name = types[i].getName();
- MobyServiceType type = MobyServiceType.getServiceType(name, reg);
- if (type != null)
- list.add(type);
+ File file = new File (serviceTypesCache, name);
+ try {
+ types[i].setParentNames (createServiceTypeRelationshipsFromXML (load (file)));
+ } catch (MobyException e) {
+ log.error (MSG_CACHE_BAD_FILE (file, e));
+ }
}
- return list.toArray(new MobyServiceType[]{});
+ return types;
}
- }
+}
/***************************************************************************
* parse list file for just the names of the Namespaces
@@ -1026,33 +984,11 @@
return results;
}
- protected static boolean rdfExists(File cache) {
- File rdfFile = new File(cache, RDF_FILE);
- return rdfFile.exists();
- }
-
/***************************************************************************
* Some file (when a cache is being tested for emptyness) are ignored.
**************************************************************************/
- @Override protected static boolean ignoredForEmptiness (File file) {
+ protected static boolean ignoredForEmptiness (File file) {
String path = file.getPath();
- return path.endsWith("~") || path.endsWith("__R__D__F__");
- }
-
- private boolean isNamespacesLoaded() {
- return namespacesLoaded;
- }
-
- private void setNamespacesLoaded(boolean namespacesLoaded) {
- this.namespacesLoaded = namespacesLoaded;
- }
-
- private boolean isServiceTypesLoaded() {
- return serviceTypesLoaded;
+ return path.endsWith("~");
}
-
- private void setServiceTypesLoaded(boolean serviceTypesLoaded) {
- this.serviceTypesLoaded = serviceTypesLoaded;
- }
-
}
More information about the MOBY-guts
mailing list