svcore  1.9
PluginRDFIndexer.cpp
Go to the documentation of this file.
00001 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
00002 
00003 /*
00004     Sonic Visualiser
00005     An audio file viewer and annotation editor.
00006     Centre for Digital Music, Queen Mary, University of London.
00007     This file copyright 2008-2012 QMUL.
00008    
00009     This program is free software; you can redistribute it and/or
00010     modify it under the terms of the GNU General Public License as
00011     published by the Free Software Foundation; either version 2 of the
00012     License, or (at your option) any later version.  See the file
00013     COPYING included with this distribution for more information.
00014 */
00015 
00016 #include "PluginRDFIndexer.h"
00017 
00018 #include "data/fileio/CachedFile.h"
00019 #include "data/fileio/FileSource.h"
00020 #include "data/fileio/PlaylistFileReader.h"
00021 #include "plugin/PluginIdentifier.h"
00022 
00023 #include "base/Profiler.h"
00024 
00025 #include <vamp-hostsdk/PluginHostAdapter.h>
00026 
00027 #include <dataquay/BasicStore.h>
00028 #include <dataquay/RDFException.h>
00029 
00030 #include <QFileInfo>
00031 #include <QDir>
00032 #include <QUrl>
00033 #include <QDateTime>
00034 #include <QSettings>
00035 #include <QFile>
00036 
00037 #include <iostream>
00038 
00039 using std::vector;
00040 using std::string;
00041 using Vamp::PluginHostAdapter;
00042 
00043 using Dataquay::Uri;
00044 using Dataquay::Node;
00045 using Dataquay::Nodes;
00046 using Dataquay::Triple;
00047 using Dataquay::Triples;
00048 using Dataquay::BasicStore;
00049 using Dataquay::RDFException;
00050 using Dataquay::RDFDuplicateImportException;
00051 
00052 PluginRDFIndexer *
00053 PluginRDFIndexer::m_instance = 0;
00054 
00055 PluginRDFIndexer *
00056 PluginRDFIndexer::getInstance() 
00057 {
00058     if (!m_instance) m_instance = new PluginRDFIndexer();
00059     return m_instance;
00060 }
00061 
00062 PluginRDFIndexer::PluginRDFIndexer() :
00063     m_index(new Dataquay::BasicStore)
00064 {
00065     m_index->addPrefix("vamp", Uri("http://purl.org/ontology/vamp/"));
00066     m_index->addPrefix("foaf", Uri("http://xmlns.com/foaf/0.1/"));
00067     m_index->addPrefix("dc", Uri("http://purl.org/dc/elements/1.1/"));
00068     indexInstalledURLs();
00069 }
00070 
00071 const BasicStore *
00072 PluginRDFIndexer::getIndex()
00073 {
00074     return m_index;
00075 }
00076 
00077 PluginRDFIndexer::~PluginRDFIndexer()
00078 {
00079     QMutexLocker locker(&m_mutex);
00080 }
00081 
00082 void
00083 PluginRDFIndexer::indexInstalledURLs()
00084 {
00085     vector<string> paths = PluginHostAdapter::getPluginPath();
00086 
00087 //    cerr << "\nPluginRDFIndexer::indexInstalledURLs: pid is " << getpid() << endl;
00088 
00089     QStringList filters;
00090     filters << "*.ttl";
00091     filters << "*.TTL";
00092     filters << "*.n3";
00093     filters << "*.N3";
00094     filters << "*.rdf";
00095     filters << "*.RDF";
00096 
00097     // Search each Vamp plugin path for an RDF file that either has
00098     // name "soname", "soname:label" or "soname/label" plus RDF
00099     // extension.  Use that order of preference, and prefer ttl over
00100     // n3 over rdf extension.
00101 
00102     for (vector<string>::const_iterator i = paths.begin(); i != paths.end(); ++i) {
00103 
00104         QDir dir(i->c_str());
00105         if (!dir.exists()) continue;
00106 
00107         QStringList entries = dir.entryList
00108             (filters, QDir::Files | QDir::Readable);
00109 
00110         for (QStringList::const_iterator j = entries.begin();
00111              j != entries.end(); ++j) {
00112 
00113             QFileInfo fi(dir.filePath(*j));
00114             pullFile(fi.absoluteFilePath());
00115         }
00116 
00117         QStringList subdirs = dir.entryList
00118             (QDir::AllDirs | QDir::NoDotAndDotDot | QDir::Readable);
00119 
00120         for (QStringList::const_iterator j = subdirs.begin();
00121              j != subdirs.end(); ++j) {
00122 
00123             QDir subdir(dir.filePath(*j));
00124             if (subdir.exists()) {
00125                 entries = subdir.entryList
00126                     (filters, QDir::Files | QDir::Readable);
00127                 for (QStringList::const_iterator k = entries.begin();
00128                      k != entries.end(); ++k) {
00129                     QFileInfo fi(subdir.filePath(*k));
00130                     pullFile(fi.absoluteFilePath());
00131                 }
00132             }
00133         }
00134     }
00135 
00136     reindex();
00137 }
00138 
00139 bool
00140 PluginRDFIndexer::indexConfiguredURLs()
00141 {
00142     SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs" << endl;
00143 
00144     QSettings settings;
00145     settings.beginGroup("RDF");
00146     
00147     QString indexKey("rdf-indices");
00148     QStringList indices = settings.value(indexKey).toStringList();
00149     
00150     for (int i = 0; i < indices.size(); ++i) {
00151 
00152         QString index = indices[i];
00153 
00154         SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs: index url is "
00155                   << index << endl;
00156 
00157         CachedFile cf(index);
00158         if (!cf.isOK()) continue;
00159 
00160         FileSource indexSource(cf.getLocalFilename());
00161 
00162         PlaylistFileReader reader(indexSource);
00163         if (!reader.isOK()) continue;
00164 
00165         PlaylistFileReader::Playlist list = reader.load();
00166         for (PlaylistFileReader::Playlist::const_iterator j = list.begin();
00167              j != list.end(); ++j) {
00168             SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs: url is "
00169                   << *j << endl;
00170             pullURL(*j);
00171         }
00172     }
00173 
00174     QString urlListKey("rdf-urls");
00175     QStringList urls = settings.value(urlListKey).toStringList();
00176 
00177     for (int i = 0; i < urls.size(); ++i) {
00178         pullURL(urls[i]);
00179     }
00180     
00181     settings.endGroup();
00182     reindex();
00183     return true;
00184 }
00185 
00186 QString
00187 PluginRDFIndexer::getURIForPluginId(QString pluginId)
00188 {
00189     QMutexLocker locker(&m_mutex);
00190 
00191     if (m_idToUriMap.find(pluginId) == m_idToUriMap.end()) return "";
00192     return m_idToUriMap[pluginId];
00193 }
00194 
00195 QString
00196 PluginRDFIndexer::getIdForPluginURI(QString uri)
00197 {
00198     m_mutex.lock();
00199 
00200     if (m_uriToIdMap.find(uri) == m_uriToIdMap.end()) {
00201 
00202         m_mutex.unlock();
00203 
00204         // Haven't found this uri referenced in any document on the
00205         // local filesystem; try resolving the pre-fragment part of
00206         // the uri as a document URL and reading that if possible.
00207 
00208         // Because we may want to refer to this document again, we
00209         // cache it locally if it turns out to exist.
00210 
00211         cerr << "PluginRDFIndexer::getIdForPluginURI: NOTE: Failed to find a local RDF document describing plugin <" << uri << ">: attempting to retrieve one remotely by guesswork" << endl;
00212 
00213         QString baseUrl = QUrl(uri).toString(QUrl::RemoveFragment);
00214 
00215         indexURL(baseUrl);
00216 
00217         m_mutex.lock();
00218 
00219         if (m_uriToIdMap.find(uri) == m_uriToIdMap.end()) {
00220             m_uriToIdMap[uri] = "";
00221         }
00222     }
00223 
00224     QString id = m_uriToIdMap[uri];
00225     m_mutex.unlock();
00226     return id;
00227 }
00228 
00229 QStringList
00230 PluginRDFIndexer::getIndexedPluginIds() 
00231 {
00232     QMutexLocker locker(&m_mutex);
00233 
00234     QStringList ids;
00235     for (StringMap::const_iterator i = m_idToUriMap.begin();
00236          i != m_idToUriMap.end(); ++i) {
00237         ids.push_back(i->first);
00238     }
00239     return ids;
00240 }
00241 
00242 bool
00243 PluginRDFIndexer::pullFile(QString filepath)
00244 {
00245     QUrl url = QUrl::fromLocalFile(filepath);
00246     QString urlString = url.toString();
00247     return pullURL(urlString);
00248 }
00249 
00250 bool
00251 PluginRDFIndexer::indexURL(QString urlString)
00252 {
00253     bool pulled = pullURL(urlString);
00254     if (!pulled) return false;
00255     reindex();
00256     return true;
00257 }
00258 
00259 bool
00260 PluginRDFIndexer::pullURL(QString urlString)
00261 {
00262     Profiler profiler("PluginRDFIndexer::indexURL");
00263 
00264 //    cerr << "PluginRDFIndexer::indexURL(" << urlString << ")" << endl;
00265 
00266     QMutexLocker locker(&m_mutex);
00267 
00268     QUrl local = urlString;
00269 
00270     if (FileSource::isRemote(urlString) &&
00271         FileSource::canHandleScheme(urlString)) {
00272 
00273         CachedFile cf(urlString, 0, "application/rdf+xml");
00274         if (!cf.isOK()) {
00275             return false;
00276         }
00277 
00278         local = QUrl::fromLocalFile(cf.getLocalFilename());
00279 
00280     } else if (urlString.startsWith("file:")) {
00281 
00282         local = QUrl(urlString);
00283 
00284     } else {
00285 
00286         local = QUrl::fromLocalFile(urlString);
00287     }
00288 
00289     try {
00290         m_index->import(local, BasicStore::ImportFailOnDuplicates);
00291     } catch (RDFDuplicateImportException &e) {
00292         cerr << e.what() << endl;
00293         cerr << "PluginRDFIndexer::pullURL: Document at " << urlString
00294              << " duplicates triples found in earlier loaded document -- skipping it" << endl;
00295         return false;
00296     } catch (RDFException &e) {
00297         cerr << e.what() << endl;
00298         cerr << "PluginRDFIndexer::pullURL: Failed to import document from "
00299              << urlString << ": " << e.what() << endl;
00300         return false;
00301     }
00302     return true;
00303 }
00304 
00305 bool
00306 PluginRDFIndexer::reindex()
00307 {
00308     Triples tt = m_index->match
00309         (Triple(Node(), Uri("a"), m_index->expand("vamp:Plugin")));
00310     Nodes plugins = tt.subjects();
00311 
00312     bool foundSomething = false;
00313     bool addedSomething = false;
00314 
00315     foreach (Node plugin, plugins) {
00316         
00317         if (plugin.type != Node::URI) {
00318             cerr << "PluginRDFIndexer::reindex: Plugin has no URI: node is "
00319                  << plugin << endl;
00320             continue;
00321         }
00322         
00323         Node idn = m_index->complete
00324             (Triple(plugin, m_index->expand("vamp:identifier"), Node()));
00325 
00326         if (idn.type != Node::Literal) {
00327             cerr << "PluginRDFIndexer::reindex: Plugin " << plugin
00328                  << " lacks vamp:identifier literal" << endl;
00329             continue;
00330         }
00331 
00332         Node libn = m_index->complete
00333             (Triple(Node(), m_index->expand("vamp:available_plugin"), plugin));
00334 
00335         if (libn.type != Node::URI) {
00336             cerr << "PluginRDFIndexer::reindex: Plugin " << plugin 
00337                  << " is not vamp:available_plugin in any library" << endl;
00338             continue;
00339         }
00340 
00341         Node son = m_index->complete
00342             (Triple(libn, m_index->expand("vamp:identifier"), Node()));
00343 
00344         if (son.type != Node::Literal) {
00345             cerr << "PluginRDFIndexer::reindex: Library " << libn
00346                  << " lacks vamp:identifier for soname" << endl;
00347             continue;
00348         }
00349 
00350         QString pluginUri = plugin.value;
00351         QString identifier = idn.value;
00352         QString soname = son.value;
00353 
00354         QString pluginId = PluginIdentifier::createIdentifier
00355             ("vamp", soname, identifier);
00356 
00357         foundSomething = true;
00358 
00359         if (m_idToUriMap.find(pluginId) != m_idToUriMap.end()) {
00360             continue;
00361         }
00362 
00363         m_idToUriMap[pluginId] = pluginUri;
00364 
00365         addedSomething = true;
00366 
00367         if (pluginUri != "") {
00368             if (m_uriToIdMap.find(pluginUri) != m_uriToIdMap.end()) {
00369                 cerr << "PluginRDFIndexer::reindex: WARNING: Found multiple plugins with the same URI:" << endl;
00370                 cerr << "  1. Plugin id \"" << m_uriToIdMap[pluginUri] << "\"" << endl;
00371                 cerr << "  2. Plugin id \"" << pluginId << "\"" << endl;
00372                 cerr << "both claim URI <" << pluginUri << ">" << endl;
00373             } else {
00374                 m_uriToIdMap[pluginUri] = pluginId;
00375             }
00376         }
00377     }
00378 
00379     if (!foundSomething) {
00380         cerr << "PluginRDFIndexer::reindex: NOTE: Plugins found, but none sufficiently described" << endl;
00381     }
00382     
00383     return addedSomething;
00384 }