X-Git-Url: http://erislabs.net/gitweb/?a=blobdiff_plain;f=source%2Fmircoders%2Fproducer%2FIndexingProducerNode.java;h=ffd6f8e88b275203fc0886084b3df23d12c67601;hb=515dab169a24634516794d8b8da80116b8e82cc9;hp=22b3a196b3ee9215e1ca6b66e29449cb6c12406d;hpb=0d512e810fc3838b6787c2a866864481d79c93ca;p=mir.git diff --git a/source/mircoders/producer/IndexingProducerNode.java b/source/mircoders/producer/IndexingProducerNode.java index 22b3a196..ffd6f8e8 100755 --- a/source/mircoders/producer/IndexingProducerNode.java +++ b/source/mircoders/producer/IndexingProducerNode.java @@ -18,55 +18,50 @@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA * * In addition, as a special exception, The Mir-coders gives permission to link - * the code of this program with any library licensed under the Apache Software License, - * The Sun (tm) Java Advanced Imaging library (JAI), The Sun JIMI library - * (or with modified versions of the above that use the same license as the above), - * and distribute linked combinations including the two. You must obey the - * GNU General Public License in all respects for all of the code used other than - * the above mentioned libraries. If you modify this file, you may extend this - * exception to your version of the file, but you are not obligated to do so. + * the code of this program with any library licensed under the Apache Software License, + * The Sun (tm) Java Advanced Imaging library (JAI), The Sun JIMI library + * (or with modified versions of the above that use the same license as the above), + * and distribute linked combinations including the two. You must obey the + * GNU General Public License in all respects for all of the code used other than + * the above mentioned libraries. If you modify this file, you may extend this + * exception to your version of the file, but you are not obligated to do so. * If you do not wish to do so, delete this exception statement from your version. */ package mircoders.producer; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; -import java.util.GregorianCalendar; -import java.util.Map; - import mir.entity.Entity; import mir.entity.adapter.EntityAdapter; import mir.log.LoggerWrapper; import mir.misc.StringUtil; +import mir.producer.AbstractProducerNode; import mir.producer.ProducerFailure; -import mir.producer.ProducerNode; +import mir.util.FileRoutines; import mir.util.ParameterExpander; import mircoders.entity.EntityContent; -import mircoders.search.AudioSearchTerm; -import mircoders.search.ContentSearchTerm; -import mircoders.search.ImagesSearchTerm; -import mircoders.search.IndexUtil; -import mircoders.search.KeywordSearchTerm; -import mircoders.search.TextSearchTerm; -import mircoders.search.TopicSearchTerm; -import mircoders.search.UnIndexedSearchTerm; -import mircoders.search.VideoSearchTerm; - +import mircoders.search.*; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.store.FSDirectory; +import java.io.File; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.Map; + -public class IndexingProducerNode implements ProducerNode { +public class IndexingProducerNode extends AbstractProducerNode { private String contentKey; private String indexPath; + private File indexBasePath; - public IndexingProducerNode(String aContentKey, String pathToIndex) { + public IndexingProducerNode(File anIndexBasePath, String aContentKey, String pathToIndex) { contentKey = aContentKey; indexPath = pathToIndex; + indexBasePath = anIndexBasePath; } public void produce(Map aValueMap, String aVerb, LoggerWrapper aLogger) @@ -74,14 +69,14 @@ public class IndexingProducerNode implements ProducerNode { IndexWriter indexWriter = null; Object data; Entity entity; - String index = null; long startTime; long endTime; + File indexFile = null; startTime = System.currentTimeMillis(); try { - index = ParameterExpander.expandExpression(aValueMap, indexPath); + indexFile = FileRoutines.getAbsoluteOrRelativeFile(indexBasePath, ParameterExpander.expandExpression(aValueMap, indexPath)); data = ParameterExpander.findValueForKey(aValueMap, contentKey); if (!(data instanceof EntityAdapter)) { @@ -98,22 +93,19 @@ public class IndexingProducerNode implements ProducerNode { entity.getClass().getName() + " adapter", null); } - aLogger.info("Indexing " + (String) entity.getValue("id") + " into " + - index); + aLogger.debug("Indexing " + entity.getFieldValue("id") + " into " + indexFile.getAbsolutePath()); // create an index here if one did not already exist - if (!(IndexReader.indexExists(index))) { - aLogger.error("Didn't find existing index, so I'm making one in " + - index); + if (!(IndexReader.indexExists(indexFile))) { + aLogger.warn("Didn't find existing index, so I'm making one in " + indexFile.getAbsolutePath()); - IndexWriter indexCreator = - new IndexWriter(index, new StandardAnalyzer(), true); + IndexWriter indexCreator = new IndexWriter(indexFile, new StandardAnalyzer(), true); indexCreator.close(); } - IndexUtil.unindexEntity((EntityContent) entity, index); + IndexUtil.unindexEntity((EntityContent) entity, indexFile); - indexWriter = new IndexWriter(index, new StandardAnalyzer(), false); + indexWriter = new IndexWriter(indexFile, new StandardAnalyzer(), false); Document theDoc = new Document(); @@ -124,10 +116,10 @@ public class IndexingProducerNode implements ProducerNode { //this initialization should go somewhere global like an xml file.... (new KeywordSearchTerm("id", "", "id", "", "id")).index(theDoc, entity); - String textValue = entity.getValue("webdb_create"); + String textValue = entity.getFieldValue("webdb_create"); Calendar calendar = GregorianCalendar.getInstance(); - int year; - int month; + int year=0; + int month=0; int day; int hours; int minutes; @@ -136,30 +128,29 @@ public class IndexingProducerNode implements ProducerNode { if (textValue!=null) { try { - year = Integer.parseInt(textValue.substring(0,4)); - month = Integer.parseInt(textValue.substring(5,7)); - day = Integer.parseInt(textValue.substring(8,10)); - hours = Integer.parseInt(textValue.substring(11,13)); - minutes = Integer.parseInt(textValue.substring(14,16)); + year = Integer.parseInt(textValue.substring(0, 4)); + month = Integer.parseInt(textValue.substring(5, 7)); + day = Integer.parseInt(textValue.substring(8, 10)); + hours = Integer.parseInt(textValue.substring(11, 13)); + minutes = Integer.parseInt(textValue.substring(14, 16)); - calendar.set(year, month-1, day, hours, minutes); + calendar.set(year, month - 1, day, hours, minutes); date = calendar.getTime(); - SimpleDateFormat formatter = new SimpleDateFormat ("yyyy.MM.dd hh:mm"); - formattedDate=formatter.format(date); + SimpleDateFormat formatter = new SimpleDateFormat("yyyy.MM.dd hh:mm"); + formattedDate = formatter.format(date); + } + catch (Throwable t) { + aLogger.warn("Error while generating content date to index", t); } - catch (Throwable t){ - aLogger.error("Error while generating content date to index: " + t.getMessage()); - t.printStackTrace(aLogger.asPrintWriter(LoggerWrapper.DEBUG_MESSAGE)); - } } (new KeywordSearchTerm("webdb_create_formatted", "search_date", "webdb_create_formatted", "webdb_create_formatted", "webdb_create_formatted")).indexValue(theDoc,formattedDate); - + (new UnIndexedSearchTerm("", "", "", "where", "where")).indexValue(theDoc, - StringUtil.webdbDate2path(entity.getValue("date")) + - entity.getValue("id") + ".shtml"); + "/"+ year +"/" + month + + entity.getFieldValue("id") + ".shtml"); (new TextSearchTerm("creator", "search_creator", "creator", "creator", "creator")).index(theDoc, entity); @@ -171,8 +162,8 @@ public class IndexingProducerNode implements ProducerNode { "creationDate", "creationDate", "creationDate")).index(theDoc, entity); (new ContentSearchTerm("content_data", "search_content", "content", "", "")).indexValue(theDoc, - entity.getValue("content_data") + " " + entity.getValue("description") + - " " + entity.getValue("title")); + entity.getFieldValue("content_data") + " " + entity.getFieldValue("description") + + " " + entity.getFieldValue("title")); (new TopicSearchTerm()).index(theDoc, entity); @@ -198,31 +189,34 @@ public class IndexingProducerNode implements ProducerNode { indexWriter.addDocument(theDoc); } catch (Throwable t) { - aLogger.error("Error while indexing content: " + t.getMessage()); - t.printStackTrace(aLogger.asPrintWriter(LoggerWrapper.DEBUG_MESSAGE)); + aLogger.error("Error while indexing content: " + t.getMessage(), t); } finally { if (indexWriter != null) { try { indexWriter.close(); - } catch (Throwable t) { - aLogger.warn("Error while closing indexWriter: " + t.getMessage()); + } + catch (Throwable t) { + aLogger.warn("Error while closing indexWriter", t); } } - try { - FSDirectory theIndexDir = FSDirectory.getDirectory(index, false); + if (indexFile!=null) { + try { + FSDirectory theIndexDir = FSDirectory.getDirectory(indexFile, false); - if (IndexReader.isLocked(theIndexDir)) { - IndexReader.unlock(theIndexDir); + if (IndexReader.isLocked(theIndexDir)) { + IndexReader.unlock(theIndexDir); + } + } + catch (Throwable t) { + aLogger.warn("Error while unlocking index", t); } - } catch (Throwable t) { - aLogger.warn("Error while unlocking index: " + t.getMessage()); } } endTime = System.currentTimeMillis(); - aLogger.info(" IndexTime: " + (endTime - startTime) + " ms
"); + aLogger.debug(" IndexTime: " + (endTime - startTime) + " ms
"); } }