Index: web/pom.xml =================================================================== --- web/pom.xml (revision 8189) +++ web/pom.xml (working copy) @@ -306,12 +306,20 @@ opendap 2.1 + + + org.jsoup + jsoup + 0.2.2 + + ${project.groupId} geonetwork-client ${project.version} provided + Index: web/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/Harvester.java =================================================================== --- web/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/Harvester.java (revision 8189) +++ web/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/Harvester.java (working copy) @@ -26,11 +26,13 @@ import jeeves.resources.dbms.Dbms; import jeeves.server.UserSession; import jeeves.server.context.ServiceContext; +import jeeves.utils.Log; import jeeves.utils.Xml; import org.fao.geonet.GeonetContext; import org.fao.geonet.constants.Geonet; import org.fao.geonet.kernel.DataManager; +import org.fao.geonet.kernel.SchemaManager; import org.fao.geonet.kernel.harvest.harvester.CategoryMapper; import org.fao.geonet.kernel.harvest.harvester.GroupMapper; import org.fao.geonet.kernel.harvest.harvester.Privileges; @@ -62,6 +64,7 @@ GeonetContext gc = (GeonetContext) context.getHandlerContext(Geonet.CONTEXT_NAME); dataMan = gc.getDataManager(); + schemaMan = gc.getSchemamanager (); } //--------------------------------------------------------------------------- @@ -72,7 +75,14 @@ public WebDavResult harvest() throws Exception { log.debug("Retrieving remote metadata information for : "+ params.name); - RemoteRetriever rr = new WebDavRetriever(); + + RemoteRetriever rr = null; + if(params.subtype.equals("webdav")) + rr = new WebDavRetriever(); + else if(params.subtype.equals("waf")) + rr = new WAFRetriever(); + + Log.info(Log.SERVICE, "webdav harvest subtype : "+params.subtype); rr.init(log, context, params); List files = rr.retrieve(); log.debug("Remote files found : "+ files.size()); @@ -153,7 +163,14 @@ log.debug(" - Setting uuid for metadata with remote path : "+ rf.getPath()); //--- set uuid inside metadata and get new xml - md = dataMan.setUUID(schema, uuid, md); + try + { + md = dataMan.setUUID(schema, uuid, md); + } + catch(Exception e) + { + return; + } log.debug(" - Adding metadata with remote path : "+ rf.getPath()); @@ -185,7 +202,7 @@ private Element retrieveMetadata(RemoteFile rf) { try { log.debug("Getting remote file : "+ rf.getPath()); - Element md = rf.getMetadata(); + Element md = rf.getMetadata(schemaMan); log.debug("Record got:\n"+ Xml.getString(md)); String schema = dataMan.autodetectSchema(md); @@ -324,6 +341,7 @@ private GroupMapper localGroups; private UriMapper localUris; private WebDavResult result; + private SchemaManager schemaMan; } //============================================================================= @@ -339,7 +357,7 @@ interface RemoteFile { public String getPath(); public String getChangeDate(); - public Element getMetadata() throws JDOMException, IOException, Exception; + public Element getMetadata(SchemaManager schemaMan) throws JDOMException, IOException, Exception; public boolean isMoreRecentThan(String localDate); } Index: web/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WAFRemoteFile.java =================================================================== --- web/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WAFRemoteFile.java (revision 0) +++ web/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WAFRemoteFile.java (revision 0) @@ -0,0 +1,172 @@ +//============================================================================= +//=== Copyright (C) 2001-2007 Food and Agriculture Organization of the +//=== United Nations (FAO-UN), United Nations World Food Programme (WFP) +//=== and United Nations Environment Programme (UNEP) +//=== +//=== This program is free software; you can redistribute it and/or modify +//=== it under the terms of the GNU General Public License as published by +//=== the Free Software Foundation; either version 2 of the License, or (at +//=== your option) any later version. +//=== +//=== This program is distributed in the hope that it will be useful, but +//=== WITHOUT ANY WARRANTY; without even the implied warranty of +//=== MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +//=== General Public License for more details. +//=== +//=== You should have received a copy of the GNU General Public License +//=== along with this program; if not, write to the Free Software +//=== Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA +//=== +//=== Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2, +//=== Rome - Italy. email: geonetwork@osgeo.org +//============================================================================== + +package org.fao.geonet.kernel.harvest.harvester.webdav; + +import jeeves.utils.Util; +import jeeves.utils.Xml; +import org.apache.commons.httpclient.HttpException; +import org.apache.webdav.lib.WebdavResource; +import org.fao.geonet.constants.Geonet; +import org.fao.geonet.kernel.SchemaManager; +import org.jdom.Element; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; + +//============================================================================= + +class WAFRemoteFile implements RemoteFile { + + public class WXS{ + public static final String WMS = "SERVICE=WMS"; + public static final String WCS = "SERVICE=WCS"; + public static final String WFS = "SERVICE=WFS"; + public static final String WPS = "SERVICE=WPS"; + } + + //--------------------------------------------------------------------------- + //--- + //--- Constructor + //--- + //--------------------------------------------------------------------------- + + public WAFRemoteFile(String path) { + this.path = path; + } + + //--------------------------------------------------------------------------- + //--- + //--- RemoteFile interface + //--- + //--------------------------------------------------------------------------- + + public Element getMetadata(SchemaManager schemaMan) throws Exception { + try { + String type = WAFRetriever.getFileType(this.path); + if(type.equals(WAFRetriever.type_GetCapabilities)) + return getMdFromService(path, schemaMan); + else if(type.equals(WAFRetriever.type_xml)) + return Xml.loadFile(new URL(path)); + else + return null; + } + catch (HttpException x) { + throw new Exception("HTTPException : " + x.getMessage()); + } + } + + + + //--------------------------------------------------------------------------- + //--- + //--- Get Metadata from GetCapabilities file (Service) + //--- + //--------------------------------------------------------------------------- + + private Element getMdFromService(String url, SchemaManager schemaMan) throws Exception + { + Element el = null; + String styleSheet = getStyleSheet(url, schemaMan); + if(styleSheet == null) + return null; + try { + Element xml = Xml.loadFile(new URL(url)); + + // md5 the full capabilities URL + String uuid = Util.scramble (url); // is the service identifier + + Map param = new HashMap(); + param.put("uuid", uuid); + + el = Xml.transform (xml, styleSheet, param); + } catch (HttpException x) { + // TODO Auto-generated catch block + throw new Exception("HTTPException : " + x.getMessage()); + } + return el; + } + + private String getStyleSheet(String url, SchemaManager schemaMan) + { + String serviceType = getServiceType(url); + if(serviceType == null) + return null; + String styleSheet = schemaMan.getSchemaDir(outputSchema) + + Geonet.Path.CONVERT_STYLESHEETS + + "/OGCWxSGetCapabilitiesto19119/" + + "/OGC" + + serviceType + + "GetCapabilities-to-ISO19119_ISO19139.xsl"; + return styleSheet; + } + + //--------------------------------------------------------------------------- + //--- + //--- Get Service Type + //--- + //--------------------------------------------------------------------------- + + private String getServiceType(String url) + { + if(url.toUpperCase().contains(WXS.WMS)) + return "WMS"; + else if(url.toUpperCase().contains(WXS.WCS)) + return "WCS"; + else if(url.toUpperCase().contains(WXS.WFS)) + return "WFS"; + else if(url.toUpperCase().contains(WXS.WPS)) + return "WPS"; + else + return null; + } + + //--------------------------------------------------------------------------- + + public boolean isMoreRecentThan(String localChangeDate) { + return true; + } + + //--------------------------------------------------------------------------- + //--- + //--- Variables + //--- + //--------------------------------------------------------------------------- + + private String path; + private static String outputSchema = "iso19139"; + @Override + public String getPath() { + // TODO Auto-generated method stub + return path; + } + + @Override + public String getChangeDate() { + // TODO Auto-generated method stub + return null; + } + +} + +//============================================================================= \ No newline at end of file Index: web/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WAFRetriever.java =================================================================== --- web/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WAFRetriever.java (revision 0) +++ web/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WAFRetriever.java (revision 0) @@ -0,0 +1,114 @@ +//============================================================================= +//=== Copyright (C) 2001-2007 Food and Agriculture Organization of the +//=== United Nations (FAO-UN), United Nations World Food Programme (WFP) +//=== and United Nations Environment Programme (UNEP) +//=== +//=== This program is free software; you can redistribute it and/or modify +//=== it under the terms of the GNU General Public License as published by +//=== the Free Software Foundation; either version 2 of the License, or (at +//=== your option) any later version. +//=== +//=== This program is distributed in the hope that it will be useful, but +//=== WITHOUT ANY WARRANTY; without even the implied warranty of +//=== MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +//=== General Public License for more details. +//=== +//=== You should have received a copy of the GNU General Public License +//=== along with this program; if not, write to the Free Software +//=== Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA +//=== +//=== Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2, +//=== Rome - Italy. email: geonetwork@osgeo.org +//============================================================================== + +package org.fao.geonet.kernel.harvest.harvester.webdav; + +import jeeves.interfaces.Logger; +import jeeves.server.context.ServiceContext; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; + +class WAFRetriever implements RemoteRetriever { + //-------------------------------------------------------------------------- + //--- + //--- RemoteRetriever interface + //--- + //-------------------------------------------------------------------------- + + public void init(Logger log, ServiceContext context, WebDavParams params) { + this.log = log; + this.params = params; + } + + //--------------------------------------------------------------------------- + + public List retrieve() throws Exception { + + files.clear(); + retrieveFiles(params.url); + return files; + } + + //--------------------------------------------------------------------------- + + public void destroy() { + return; + } + + //--------------------------------------------------------------------------- + + private void retrieveFiles(String wafurl) throws IOException { + + log.debug("Scanning resource : "+ wafurl); + + Document doc = Jsoup.parse(new URL(wafurl),3000); + Elements links = doc.select("a[href]"); + for (Element link : links) { + String url = link.attr("abs:href"); + if(getFileType(url) != null) + files.add(new WAFRemoteFile(url)); + else + continue; + } + + } + + //--------------------------------------------------------------------------- + //--- + //--- check type if url is a xml file or GetCapabilities file + //--- + //--------------------------------------------------------------------------- + + public static String getFileType(String path) + { + if(path.toUpperCase().contains("REQUEST=GETCAPABILITIES")) + return type_GetCapabilities; + else if(path.toUpperCase().endsWith(".XML")) + return type_xml; + else + return null; + } + + //--------------------------------------------------------------------------- + //--- + //--- Variables + //--- + //--------------------------------------------------------------------------- + + private Logger log; + private WebDavParams params; + private List files = new ArrayList(); + + public static final String type_GetCapabilities = "GetCapabilities"; + public static final String type_xml = "xml"; + +} + +//============================================================================= \ No newline at end of file Index: web/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WebDavHarvester.java =================================================================== --- web/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WebDavHarvester.java (revision 8189) +++ web/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WebDavHarvester.java (working copy) @@ -121,6 +121,7 @@ settingMan.add(dbms, "id:"+siteId, "icon", params.icon); settingMan.add(dbms, "id:"+optionsId, "validate", params.validate); settingMan.add(dbms, "id:"+optionsId, "recurse", params.recurse); + settingMan.add(dbms, "id:"+optionsId, "subtype", params.subtype); } //--------------------------------------------------------------------------- Index: web/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WebDavParams.java =================================================================== --- web/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WebDavParams.java (revision 8189) +++ web/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WebDavParams.java (working copy) @@ -63,6 +63,8 @@ validate = Util.getParam(opt, "validate", false); recurse = Util.getParam(opt, "recurse", false); + + subtype = Util.getParam(opt, "subtype",""); } //--------------------------------------------------------------------------- @@ -83,6 +85,8 @@ validate = Util.getParam(opt, "validate", validate); recurse = Util.getParam(opt, "recurse", recurse); + + subtype = Util.getParam(opt, "subtype", subtype); } //--------------------------------------------------------------------------- @@ -102,6 +106,8 @@ copy.validate = validate; copy.recurse = recurse; + copy.subtype = subtype; + return copy; } @@ -116,6 +122,8 @@ public boolean validate; public boolean recurse; + + public String subtype; } //============================================================================= Index: web/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WebDavRemoteFile.java =================================================================== --- web/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WebDavRemoteFile.java (revision 8189) +++ web/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WebDavRemoteFile.java (working copy) @@ -26,6 +26,7 @@ import jeeves.utils.Xml; import org.apache.commons.httpclient.HttpException; import org.apache.webdav.lib.WebdavResource; +import org.fao.geonet.kernel.SchemaManager; import org.fao.geonet.util.ISODate; import org.jdom.Element; @@ -56,7 +57,7 @@ //--------------------------------------------------------------------------- - public Element getMetadata() throws Exception { + public Element getMetadata(SchemaManager schemaMan) throws Exception { try { wr.setPath(path); return Xml.loadStream(wr.getMethodData()); Index: web/src/main/webapp/loc/ar/xml/harvesting.xml =================================================================== --- web/src/main/webapp/loc/ar/xml/harvesting.xml (revision 8189) +++ web/src/main/webapp/loc/ar/xml/harvesting.xml (working copy) @@ -24,7 +24,7 @@ Web DAV - Web DAV server + Web Access Folder/Web DAV server Index: web/src/main/webapp/loc/ca/xml/harvesting.xml =================================================================== --- web/src/main/webapp/loc/ca/xml/harvesting.xml (revision 8189) +++ web/src/main/webapp/loc/ca/xml/harvesting.xml (working copy) @@ -24,7 +24,7 @@ Web DAV - Servidor Web DAV + Web Access Folder/Web DAV server Index: web/src/main/webapp/loc/cn/xml/harvesting.xml =================================================================== --- web/src/main/webapp/loc/cn/xml/harvesting.xml (revision 8189) +++ web/src/main/webapp/loc/cn/xml/harvesting.xml (working copy) @@ -24,7 +24,7 @@ Web DAV - Web DAV server + Web Access Folder/Web DAV server Index: web/src/main/webapp/loc/de/xml/harvesting.xml =================================================================== --- web/src/main/webapp/loc/de/xml/harvesting.xml (revision 8189) +++ web/src/main/webapp/loc/de/xml/harvesting.xml (working copy) @@ -24,7 +24,7 @@ Web DAV - Web DAV Server + Web Access Folder/Web DAV server Index: web/src/main/webapp/loc/en/xml/harvesting.xml =================================================================== --- web/src/main/webapp/loc/en/xml/harvesting.xml (revision 8189) +++ web/src/main/webapp/loc/en/xml/harvesting.xml (working copy) @@ -24,7 +24,7 @@ Web DAV - Web DAV server + Web Access Folder/Web DAV server Index: web/src/main/webapp/loc/es/xml/harvesting.xml =================================================================== --- web/src/main/webapp/loc/es/xml/harvesting.xml (revision 8189) +++ web/src/main/webapp/loc/es/xml/harvesting.xml (working copy) @@ -24,7 +24,7 @@ Web DAV - Servidor Web DAV server + Web Access Folder/Servidor Web DAV server Index: web/src/main/webapp/loc/fi/xml/harvesting.xml =================================================================== --- web/src/main/webapp/loc/fi/xml/harvesting.xml (revision 8189) +++ web/src/main/webapp/loc/fi/xml/harvesting.xml (working copy) @@ -24,7 +24,7 @@ Web DAV - Web DAV server + Web Access Folder/Web DAV server Index: web/src/main/webapp/loc/fr/xml/harvesting.xml =================================================================== --- web/src/main/webapp/loc/fr/xml/harvesting.xml (revision 8189) +++ web/src/main/webapp/loc/fr/xml/harvesting.xml (working copy) @@ -24,7 +24,7 @@ Web DAV - Serveur Web DAV + Web Access Folder/Serveur Web DAV Index: web/src/main/webapp/loc/nl/xml/harvesting.xml =================================================================== --- web/src/main/webapp/loc/nl/xml/harvesting.xml (revision 8189) +++ web/src/main/webapp/loc/nl/xml/harvesting.xml (working copy) @@ -24,7 +24,7 @@ WebDAV - WebDAV server + Web Access Folder/WebDAV server Index: web/src/main/webapp/loc/no/xml/harvesting.xml =================================================================== --- web/src/main/webapp/loc/no/xml/harvesting.xml (revision 8189) +++ web/src/main/webapp/loc/no/xml/harvesting.xml (working copy) @@ -24,7 +24,7 @@ Web DAV - Web DAV server + Web Access Folder/Web DAV server Index: web/src/main/webapp/loc/pt/xml/harvesting.xml =================================================================== --- web/src/main/webapp/loc/pt/xml/harvesting.xml (revision 8189) +++ web/src/main/webapp/loc/pt/xml/harvesting.xml (working copy) @@ -24,7 +24,7 @@ Web DAV - Servidor Web DAV + Web Access Folder/Servidor Web DAV Index: web/src/main/webapp/loc/ru/xml/harvesting.xml =================================================================== --- web/src/main/webapp/loc/ru/xml/harvesting.xml (revision 8189) +++ web/src/main/webapp/loc/ru/xml/harvesting.xml (working copy) @@ -24,7 +24,7 @@ Web DAV - Сервер Web DAV + Web Access Folder/Сервер Web DAV Index: web/src/main/webapp/scripts/harvesting/webdav/model.js =================================================================== --- web/src/main/webapp/scripts/harvesting/webdav/model.js (revision 8189) +++ web/src/main/webapp/scripts/harvesting/webdav/model.js (working copy) @@ -117,6 +117,7 @@ ' {EVERY}'+ ' {ONE_RUN_ONLY}'+ ' {RECURSE}'+ +' {SUBTYPE}'+ ' {VALIDATE}'+ ' '+ Index: web/src/main/webapp/scripts/harvesting/webdav/view.js =================================================================== --- web/src/main/webapp/scripts/harvesting/webdav/view.js (revision 8189) +++ web/src/main/webapp/scripts/harvesting/webdav/view.js (working copy) @@ -44,6 +44,7 @@ { id:'wd.name', type:'length', minSize :1, maxSize :200 }, { id:'wd.url', type:'length', minSize :1, maxSize :200 }, { id:'wd.url', type:'url' }, + { id:'wd.subtype', type:'length', minSize :1, maxSize :200 }, { id:'wd.username', type:'length', minSize :0, maxSize :200 }, { id:'wd.password', type:'length', minSize :0, maxSize :200 }, { id:'wd.every.days', type:'integer', minValue:0, maxValue:99 }, @@ -91,6 +92,7 @@ hvutil.setOption(site, 'icon', 'wd.icon'); hvutil.setOption(options, 'validate', 'wd.validate'); hvutil.setOption(options, 'recurse', 'wd.recurse'); + hvutil.setOption(options, 'subtype', 'wd.subtype'); //--- add privileges entries @@ -116,7 +118,7 @@ data.ICON = $F('wd.icon'); data.VALIDATE = $('wd.validate').checked; data.RECURSE = $('wd.recurse') .checked; - + data.SUBTYPE = $F('wd.subtype'); //--- retrieve privileges and categories information data.PRIVILEGES = this.getPrivileges(); Index: web/src/main/webapp/xsl/harvesting/webdav/webdav.xsl =================================================================== --- web/src/main/webapp/xsl/harvesting/webdav/webdav.xsl (revision 8189) +++ web/src/main/webapp/xsl/harvesting/webdav/webdav.xsl (working copy) @@ -26,6 +26,17 @@

+ + + + + + @@ -75,6 +86,7 @@

Subtype + +
+
Index: web/src/main/webapp/xsl/xml/harvesting/webdav.xsl =================================================================== --- web/src/main/webapp/xsl/xml/harvesting/webdav.xsl (revision 8189) +++ web/src/main/webapp/xsl/xml/harvesting/webdav.xsl (working copy) @@ -20,6 +20,7 @@ +