svcore
1.9
|
00001 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ 00002 00003 /* 00004 Sonic Visualiser 00005 An audio file viewer and annotation editor. 00006 Centre for Digital Music, Queen Mary, University of London. 00007 This file copyright 2008-2012 QMUL. 00008 00009 This program is free software; you can redistribute it and/or 00010 modify it under the terms of the GNU General Public License as 00011 published by the Free Software Foundation; either version 2 of the 00012 License, or (at your option) any later version. See the file 00013 COPYING included with this distribution for more information. 00014 */ 00015 00016 #include "PluginRDFIndexer.h" 00017 00018 #include "data/fileio/CachedFile.h" 00019 #include "data/fileio/FileSource.h" 00020 #include "data/fileio/PlaylistFileReader.h" 00021 #include "plugin/PluginIdentifier.h" 00022 00023 #include "base/Profiler.h" 00024 00025 #include <vamp-hostsdk/PluginHostAdapter.h> 00026 00027 #include <dataquay/BasicStore.h> 00028 #include <dataquay/RDFException.h> 00029 00030 #include <QFileInfo> 00031 #include <QDir> 00032 #include <QUrl> 00033 #include <QDateTime> 00034 #include <QSettings> 00035 #include <QFile> 00036 00037 #include <iostream> 00038 00039 using std::vector; 00040 using std::string; 00041 using Vamp::PluginHostAdapter; 00042 00043 using Dataquay::Uri; 00044 using Dataquay::Node; 00045 using Dataquay::Nodes; 00046 using Dataquay::Triple; 00047 using Dataquay::Triples; 00048 using Dataquay::BasicStore; 00049 using Dataquay::RDFException; 00050 using Dataquay::RDFDuplicateImportException; 00051 00052 PluginRDFIndexer * 00053 PluginRDFIndexer::m_instance = 0; 00054 00055 PluginRDFIndexer * 00056 PluginRDFIndexer::getInstance() 00057 { 00058 if (!m_instance) m_instance = new PluginRDFIndexer(); 00059 return m_instance; 00060 } 00061 00062 PluginRDFIndexer::PluginRDFIndexer() : 00063 m_index(new Dataquay::BasicStore) 00064 { 00065 m_index->addPrefix("vamp", Uri("http://purl.org/ontology/vamp/")); 00066 m_index->addPrefix("foaf", Uri("http://xmlns.com/foaf/0.1/")); 00067 m_index->addPrefix("dc", Uri("http://purl.org/dc/elements/1.1/")); 00068 indexInstalledURLs(); 00069 } 00070 00071 const BasicStore * 00072 PluginRDFIndexer::getIndex() 00073 { 00074 return m_index; 00075 } 00076 00077 PluginRDFIndexer::~PluginRDFIndexer() 00078 { 00079 QMutexLocker locker(&m_mutex); 00080 } 00081 00082 void 00083 PluginRDFIndexer::indexInstalledURLs() 00084 { 00085 vector<string> paths = PluginHostAdapter::getPluginPath(); 00086 00087 // cerr << "\nPluginRDFIndexer::indexInstalledURLs: pid is " << getpid() << endl; 00088 00089 QStringList filters; 00090 filters << "*.ttl"; 00091 filters << "*.TTL"; 00092 filters << "*.n3"; 00093 filters << "*.N3"; 00094 filters << "*.rdf"; 00095 filters << "*.RDF"; 00096 00097 // Search each Vamp plugin path for an RDF file that either has 00098 // name "soname", "soname:label" or "soname/label" plus RDF 00099 // extension. Use that order of preference, and prefer ttl over 00100 // n3 over rdf extension. 00101 00102 for (vector<string>::const_iterator i = paths.begin(); i != paths.end(); ++i) { 00103 00104 QDir dir(i->c_str()); 00105 if (!dir.exists()) continue; 00106 00107 QStringList entries = dir.entryList 00108 (filters, QDir::Files | QDir::Readable); 00109 00110 for (QStringList::const_iterator j = entries.begin(); 00111 j != entries.end(); ++j) { 00112 00113 QFileInfo fi(dir.filePath(*j)); 00114 pullFile(fi.absoluteFilePath()); 00115 } 00116 00117 QStringList subdirs = dir.entryList 00118 (QDir::AllDirs | QDir::NoDotAndDotDot | QDir::Readable); 00119 00120 for (QStringList::const_iterator j = subdirs.begin(); 00121 j != subdirs.end(); ++j) { 00122 00123 QDir subdir(dir.filePath(*j)); 00124 if (subdir.exists()) { 00125 entries = subdir.entryList 00126 (filters, QDir::Files | QDir::Readable); 00127 for (QStringList::const_iterator k = entries.begin(); 00128 k != entries.end(); ++k) { 00129 QFileInfo fi(subdir.filePath(*k)); 00130 pullFile(fi.absoluteFilePath()); 00131 } 00132 } 00133 } 00134 } 00135 00136 reindex(); 00137 } 00138 00139 bool 00140 PluginRDFIndexer::indexConfiguredURLs() 00141 { 00142 SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs" << endl; 00143 00144 QSettings settings; 00145 settings.beginGroup("RDF"); 00146 00147 QString indexKey("rdf-indices"); 00148 QStringList indices = settings.value(indexKey).toStringList(); 00149 00150 for (int i = 0; i < indices.size(); ++i) { 00151 00152 QString index = indices[i]; 00153 00154 SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs: index url is " 00155 << index << endl; 00156 00157 CachedFile cf(index); 00158 if (!cf.isOK()) continue; 00159 00160 FileSource indexSource(cf.getLocalFilename()); 00161 00162 PlaylistFileReader reader(indexSource); 00163 if (!reader.isOK()) continue; 00164 00165 PlaylistFileReader::Playlist list = reader.load(); 00166 for (PlaylistFileReader::Playlist::const_iterator j = list.begin(); 00167 j != list.end(); ++j) { 00168 SVDEBUG << "PluginRDFIndexer::indexConfiguredURLs: url is " 00169 << *j << endl; 00170 pullURL(*j); 00171 } 00172 } 00173 00174 QString urlListKey("rdf-urls"); 00175 QStringList urls = settings.value(urlListKey).toStringList(); 00176 00177 for (int i = 0; i < urls.size(); ++i) { 00178 pullURL(urls[i]); 00179 } 00180 00181 settings.endGroup(); 00182 reindex(); 00183 return true; 00184 } 00185 00186 QString 00187 PluginRDFIndexer::getURIForPluginId(QString pluginId) 00188 { 00189 QMutexLocker locker(&m_mutex); 00190 00191 if (m_idToUriMap.find(pluginId) == m_idToUriMap.end()) return ""; 00192 return m_idToUriMap[pluginId]; 00193 } 00194 00195 QString 00196 PluginRDFIndexer::getIdForPluginURI(QString uri) 00197 { 00198 m_mutex.lock(); 00199 00200 if (m_uriToIdMap.find(uri) == m_uriToIdMap.end()) { 00201 00202 m_mutex.unlock(); 00203 00204 // Haven't found this uri referenced in any document on the 00205 // local filesystem; try resolving the pre-fragment part of 00206 // the uri as a document URL and reading that if possible. 00207 00208 // Because we may want to refer to this document again, we 00209 // cache it locally if it turns out to exist. 00210 00211 cerr << "PluginRDFIndexer::getIdForPluginURI: NOTE: Failed to find a local RDF document describing plugin <" << uri << ">: attempting to retrieve one remotely by guesswork" << endl; 00212 00213 QString baseUrl = QUrl(uri).toString(QUrl::RemoveFragment); 00214 00215 indexURL(baseUrl); 00216 00217 m_mutex.lock(); 00218 00219 if (m_uriToIdMap.find(uri) == m_uriToIdMap.end()) { 00220 m_uriToIdMap[uri] = ""; 00221 } 00222 } 00223 00224 QString id = m_uriToIdMap[uri]; 00225 m_mutex.unlock(); 00226 return id; 00227 } 00228 00229 QStringList 00230 PluginRDFIndexer::getIndexedPluginIds() 00231 { 00232 QMutexLocker locker(&m_mutex); 00233 00234 QStringList ids; 00235 for (StringMap::const_iterator i = m_idToUriMap.begin(); 00236 i != m_idToUriMap.end(); ++i) { 00237 ids.push_back(i->first); 00238 } 00239 return ids; 00240 } 00241 00242 bool 00243 PluginRDFIndexer::pullFile(QString filepath) 00244 { 00245 QUrl url = QUrl::fromLocalFile(filepath); 00246 QString urlString = url.toString(); 00247 return pullURL(urlString); 00248 } 00249 00250 bool 00251 PluginRDFIndexer::indexURL(QString urlString) 00252 { 00253 bool pulled = pullURL(urlString); 00254 if (!pulled) return false; 00255 reindex(); 00256 return true; 00257 } 00258 00259 bool 00260 PluginRDFIndexer::pullURL(QString urlString) 00261 { 00262 Profiler profiler("PluginRDFIndexer::indexURL"); 00263 00264 // cerr << "PluginRDFIndexer::indexURL(" << urlString << ")" << endl; 00265 00266 QMutexLocker locker(&m_mutex); 00267 00268 QUrl local = urlString; 00269 00270 if (FileSource::isRemote(urlString) && 00271 FileSource::canHandleScheme(urlString)) { 00272 00273 CachedFile cf(urlString, 0, "application/rdf+xml"); 00274 if (!cf.isOK()) { 00275 return false; 00276 } 00277 00278 local = QUrl::fromLocalFile(cf.getLocalFilename()); 00279 00280 } else if (urlString.startsWith("file:")) { 00281 00282 local = QUrl(urlString); 00283 00284 } else { 00285 00286 local = QUrl::fromLocalFile(urlString); 00287 } 00288 00289 try { 00290 m_index->import(local, BasicStore::ImportFailOnDuplicates); 00291 } catch (RDFDuplicateImportException &e) { 00292 cerr << e.what() << endl; 00293 cerr << "PluginRDFIndexer::pullURL: Document at " << urlString 00294 << " duplicates triples found in earlier loaded document -- skipping it" << endl; 00295 return false; 00296 } catch (RDFException &e) { 00297 cerr << e.what() << endl; 00298 cerr << "PluginRDFIndexer::pullURL: Failed to import document from " 00299 << urlString << ": " << e.what() << endl; 00300 return false; 00301 } 00302 return true; 00303 } 00304 00305 bool 00306 PluginRDFIndexer::reindex() 00307 { 00308 Triples tt = m_index->match 00309 (Triple(Node(), Uri("a"), m_index->expand("vamp:Plugin"))); 00310 Nodes plugins = tt.subjects(); 00311 00312 bool foundSomething = false; 00313 bool addedSomething = false; 00314 00315 foreach (Node plugin, plugins) { 00316 00317 if (plugin.type != Node::URI) { 00318 cerr << "PluginRDFIndexer::reindex: Plugin has no URI: node is " 00319 << plugin << endl; 00320 continue; 00321 } 00322 00323 Node idn = m_index->complete 00324 (Triple(plugin, m_index->expand("vamp:identifier"), Node())); 00325 00326 if (idn.type != Node::Literal) { 00327 cerr << "PluginRDFIndexer::reindex: Plugin " << plugin 00328 << " lacks vamp:identifier literal" << endl; 00329 continue; 00330 } 00331 00332 Node libn = m_index->complete 00333 (Triple(Node(), m_index->expand("vamp:available_plugin"), plugin)); 00334 00335 if (libn.type != Node::URI) { 00336 cerr << "PluginRDFIndexer::reindex: Plugin " << plugin 00337 << " is not vamp:available_plugin in any library" << endl; 00338 continue; 00339 } 00340 00341 Node son = m_index->complete 00342 (Triple(libn, m_index->expand("vamp:identifier"), Node())); 00343 00344 if (son.type != Node::Literal) { 00345 cerr << "PluginRDFIndexer::reindex: Library " << libn 00346 << " lacks vamp:identifier for soname" << endl; 00347 continue; 00348 } 00349 00350 QString pluginUri = plugin.value; 00351 QString identifier = idn.value; 00352 QString soname = son.value; 00353 00354 QString pluginId = PluginIdentifier::createIdentifier 00355 ("vamp", soname, identifier); 00356 00357 foundSomething = true; 00358 00359 if (m_idToUriMap.find(pluginId) != m_idToUriMap.end()) { 00360 continue; 00361 } 00362 00363 m_idToUriMap[pluginId] = pluginUri; 00364 00365 addedSomething = true; 00366 00367 if (pluginUri != "") { 00368 if (m_uriToIdMap.find(pluginUri) != m_uriToIdMap.end()) { 00369 cerr << "PluginRDFIndexer::reindex: WARNING: Found multiple plugins with the same URI:" << endl; 00370 cerr << " 1. Plugin id \"" << m_uriToIdMap[pluginUri] << "\"" << endl; 00371 cerr << " 2. Plugin id \"" << pluginId << "\"" << endl; 00372 cerr << "both claim URI <" << pluginUri << ">" << endl; 00373 } else { 00374 m_uriToIdMap[pluginUri] = pluginId; 00375 } 00376 } 00377 } 00378 00379 if (!foundSomething) { 00380 cerr << "PluginRDFIndexer::reindex: NOTE: Plugins found, but none sufficiently described" << endl; 00381 } 00382 00383 return addedSomething; 00384 }