From 3389d56d4d82089033fd83f9c8758126ffd7da44 Mon Sep 17 00:00:00 2001
From: zapata <zapata>
Date: Mon, 27 May 2002 23:46:41 +0000
Subject: [PATCH] At last the EntityBatchingProducerNode is working. This will
 replace the old ProducerList. Some bugfixes were done too.

---
 source/mir/entity/EntityBrowser.java               |  52 +++---
 source/mir/entity/adapter/EntityAdapter.java       |  15 +-
 .../entity/adapter/EntityAdapterDefinition.java    |  71 +++++++--
 .../mir/entity/adapter/EntityIteratorAdapter.java  |   4 +
 source/mir/generator/FreemarkerGenerator.java      |   8 +-
 .../mir/producer/EntityBatchingProducerNode.java   | 177 ++++++++++++++++-----
 .../producer/EntityEnumeratingProducerNode.java    |   1 -
 source/mir/producer/EntityListProducerNode.java    |  13 +-
 .../entity/adapter/ContentAdapterDefinition.java   | 144 +----------------
 source/mircoders/global/JobQueue.java              |   4 +
 source/mircoders/global/ProducerEngine.java        |   1 +
 .../basic/MirBasicOpenPostingLocalizer.java        |   2 +-
 .../basic/MirBasicProducerToolLocalizer.java       |   8 +
 .../producer/ContentMarkingProducerNode.java       |  48 ++++++
 .../BoliviaProducerLocalizer.java                  |  36 ++---
 .../EHProducerLocalizer.java                       |  67 +++++++-
 .../indymedia.nl/IndyNLProducerLocalizer.java      |  22 +--
 17 files changed, 402 insertions(+), 271 deletions(-)
 create mode 100755 source/mircoders/producer/ContentMarkingProducerNode.java

diff --git a/source/mir/entity/EntityBrowser.java b/source/mir/entity/EntityBrowser.java
index 5a05e18e..a320ddee 100755
--- a/source/mir/entity/EntityBrowser.java
+++ b/source/mir/entity/EntityBrowser.java
@@ -11,47 +11,59 @@ public class EntityBrowser implements RewindableIterator {
   private String whereClause;
   private String orderByClause;
   private int batchSize;
-  private int position=0;
-
+  private int toFetch;
   private EntityList currentBatch;
 
-  public EntityBrowser(StorageObject aStorage,
-          String aWhereClause, String anOrderByClause,
-          int aBatchSize) throws StorageObjectException {
+  private int skip;
+  private int limit;
+
+  private int batchPosition;
+  private int positionInBatch;
+
+  public EntityBrowser(StorageObject aStorage, String aWhereClause, String anOrderByClause,
+                       int aBatchSize, int aLimit, int aSkip) throws StorageObjectException {
 
     storage=aStorage;
     whereClause=aWhereClause;
     orderByClause=anOrderByClause;
     batchSize=aBatchSize;
+    skip=aSkip;
+    limit=aLimit;
 
     rewind();
   }
 
+  public EntityBrowser(StorageObject aStorage,
+          String aWhereClause, String anOrderByClause,
+          int aBatchSize) throws StorageObjectException {
+    this(aStorage, aWhereClause, anOrderByClause, aBatchSize, -1, 0);
+  }
+
+  public void readCurrentBatch(int aSkip) throws StorageObjectException {
+    currentBatch = storage.selectByWhereClause(whereClause, orderByClause, aSkip, batchSize);
+    batchPosition = aSkip;
+    positionInBatch = 0;
+  }
+
   public void rewind() {
     try {
-      currentBatch = storage.selectByWhereClause(whereClause, orderByClause, 0, batchSize);
-      position=0;
+      readCurrentBatch(skip);
     }
     catch (Throwable t) {
       throw new RuntimeException(t.getMessage());
     }
   }
 
-  private void readNextBatch() throws StorageObjectException  {
-    if (currentBatch.hasNextBatch()) {
-      currentBatch = storage.selectByWhereClause(whereClause, orderByClause,
-            currentBatch.getNextBatch(), batchSize);
-      position=0;
-    }
-  }
-
   public boolean hasNext() {
     try {
-      if (position>=currentBatch.size() && currentBatch.hasNextBatch()) {
-        readNextBatch();
+      if (limit>-1 && batchPosition+positionInBatch>=skip+limit)
+        return false;
+
+      if (positionInBatch>=currentBatch.size() && currentBatch.hasNextBatch()) {
+        readCurrentBatch(batchPosition+positionInBatch);
       }
 
-      return (position<currentBatch.size());
+      return (positionInBatch<currentBatch.size());
     }
     catch (Throwable t) {
       throw new RuntimeException(t.getMessage());
@@ -61,8 +73,8 @@ public class EntityBrowser implements RewindableIterator {
   public Object next() {
     try {
       if (hasNext()) {
-        Entity result = currentBatch.elementAt(position);
-        position=position+1;
+        Entity result = currentBatch.elementAt(positionInBatch);
+        positionInBatch=positionInBatch+1;
 
         return result;
       }
diff --git a/source/mir/entity/adapter/EntityAdapter.java b/source/mir/entity/adapter/EntityAdapter.java
index 3a0b76ab..74a481e9 100755
--- a/source/mir/entity/adapter/EntityAdapter.java
+++ b/source/mir/entity/adapter/EntityAdapter.java
@@ -15,11 +15,16 @@ public class EntityAdapter implements Map {
   }
 
   public boolean containsKey(Object aKey) {
-    if (aKey instanceof String)
-      return      entity.hasValueForField((String) aKey)
-              || definition.hasCalculatedField((String) aKey);
-    else
-      return false;
+    try {
+      if (aKey instanceof String)
+        return      entity.hasValueForField((String) aKey)
+                || definition.hasCalculatedField((String) aKey)
+                || entity.getFields().contains(aKey);
+    }
+    catch (Throwable t) {
+    }
+
+    return false;
   }
 
   public boolean equals(Object anObject) {
diff --git a/source/mir/entity/adapter/EntityAdapterDefinition.java b/source/mir/entity/adapter/EntityAdapterDefinition.java
index 1c60e88a..9a888fb2 100755
--- a/source/mir/entity/adapter/EntityAdapterDefinition.java
+++ b/source/mir/entity/adapter/EntityAdapterDefinition.java
@@ -28,18 +28,22 @@ public class EntityAdapterDefinition {
     calculatedFields.put(aFieldName, aField);
   }
 
-  public void addDateField(String aDestinationFieldName, String aSourceFieldName) {
-    addCalculatedField(aDestinationFieldName, new DateField(aSourceFieldName));
+  public void addMirDateField(String aDestinationFieldName, String aSourceFieldName) {
+    addCalculatedField(aDestinationFieldName, new MirDateField(aSourceFieldName));
+  }
+
+  public void addDBDateField(String aDestinationFieldName, String aSourceFieldName) {
+    addCalculatedField(aDestinationFieldName, new DBDateField(aSourceFieldName));
   }
 
   public interface CalculatedField {
     public Object getValue(EntityAdapter anEntityAdapter);
   }
 
-  private class DateField implements CalculatedField {
+  private class MirDateField implements CalculatedField {
     private String fieldName;
 
-    public DateField(String aFieldName) {
+    public MirDateField(String aFieldName) {
       fieldName = aFieldName;
     }
 
@@ -59,24 +63,63 @@ public class EntityAdapterDefinition {
           month = Integer.parseInt(textValue.substring(4,6));
           day = Integer.parseInt(textValue.substring(6,8));
 
-          calendar.set(year, month, day);
+          calendar.set(year, month-1, day);
           date = calendar.getTime();
           ;
 
           result.put("date", date);
           result.put("formatted", new DateToMapAdapter(date));
 
-/*          result.put("year", textValue.substring(0,4));
-          result.put("month", textValue.substring(4,6));
-          result.put("day", textValue.substring(6,8));
-*/        }
+        }
+        catch (Throwable t) {
+          result=null;
+        }
+      }
+
+      return result;
+
+    }
+
+  }
+
+  private class DBDateField implements CalculatedField {
+    private String fieldName;
+
+    public DBDateField(String aFieldName) {
+      fieldName = aFieldName;
+    }
+
+    public Object getValue(EntityAdapter anEntityAdapter) {
+
+      Map result = new HashMap();
+      String textValue = anEntityAdapter.getEntity().getValue(fieldName);
+      Calendar calendar = GregorianCalendar.getInstance();
+      int year;
+      int month;
+      int day;
+      int hours;
+      int minutes;
+
+      Date date;
+
+      if (textValue!=null) {
+        try {
+          year = Integer.parseInt(textValue.substring(0,4));
+          month = Integer.parseInt(textValue.substring(5,7));
+          day = Integer.parseInt(textValue.substring(8,10));
+          hours = Integer.parseInt(textValue.substring(11,13));
+          minutes = Integer.parseInt(textValue.substring(14,16));
+
+          calendar.set(year, month-1, day, hours, minutes);
+          date = calendar.getTime();
+
+          result.put("date", date);
+          result.put("formatted", new DateToMapAdapter(date));
+          result.put("raw", textValue);
+        }
         catch (Throwable t) {
           result=null;
-/*          result.put("date", null);
-          result.put("year", null);
-          result.put("month", null);
-          result.put("day", null);
-*/        }
+        }
       }
 
       return result;
diff --git a/source/mir/entity/adapter/EntityIteratorAdapter.java b/source/mir/entity/adapter/EntityIteratorAdapter.java
index 8531625f..fe967ab2 100755
--- a/source/mir/entity/adapter/EntityIteratorAdapter.java
+++ b/source/mir/entity/adapter/EntityIteratorAdapter.java
@@ -18,6 +18,10 @@ public class EntityIteratorAdapter implements RewindableIterator {
     this(new EntityBrowser(aStorage, aWhereClause, anOrderByClause, aBatchSize), aDefinition);
   }
 
+  public EntityIteratorAdapter(StorageObject aStorage, String aWhereClause, String anOrderByClause, int aBatchSize, EntityAdapterDefinition aDefinition, int aLimit, int aSkip) throws StorageObjectException {
+    this(new EntityBrowser(aStorage, aWhereClause, anOrderByClause, aBatchSize, aLimit, aSkip), aDefinition);
+  }
+
   public boolean hasNext() {
     return iterator.hasNext();
   }
diff --git a/source/mir/generator/FreemarkerGenerator.java b/source/mir/generator/FreemarkerGenerator.java
index d4778ffb..4329ee0b 100755
--- a/source/mir/generator/FreemarkerGenerator.java
+++ b/source/mir/generator/FreemarkerGenerator.java
@@ -48,6 +48,8 @@ public class FreemarkerGenerator implements Generator {
 	    return makeMapAdapter((Map) anObject);
 	  else if (anObject instanceof Iterator)
 	    return makeIteratorAdapter((Iterator) anObject);
+	  else if (anObject instanceof List)
+	    return makeIteratorAdapter(((List) anObject).iterator());
 	  else
 	    throw new TemplateModelException("Unadaptable class: " + anObject.getClass().getName());
 	}
@@ -77,8 +79,10 @@ public class FreemarkerGenerator implements Generator {
 	    if (!valuesCache.containsKey(aKey)) {
 	      Object value = map.get(aKey);
 
-	      if (value == null)
-	        throw new TemplateModelException("MapAdapter: no key "+aKey+" available");
+//    ML: this unfortunately doesn't work, because the entity doesn't seem to store
+//        fields with null values
+//    if (value == null && !map.containsKey(aKey))
+//          throw new TemplateModelException("MapAdapter: no key "+aKey+" available");
 
 	      valuesCache.put(aKey, makeAdapter(value));
 	    }
diff --git a/source/mir/producer/EntityBatchingProducerNode.java b/source/mir/producer/EntityBatchingProducerNode.java
index 446404d6..9a865c00 100755
--- a/source/mir/producer/EntityBatchingProducerNode.java
+++ b/source/mir/producer/EntityBatchingProducerNode.java
@@ -1,4 +1,4 @@
-/*package mir.producer;
+package mir.producer;
 
 import java.util.*;
 import java.io.*;
@@ -9,69 +9,161 @@ import mir.util.*;
 
 //	abstract public int getSize(String where) throws SQLException,StorageObjectException;
 
-
 public class EntityBatchingProducerNode implements ProducerNode {
   private Map verbs;
-  private EntityBatchingProducerNodeVerb defaultVerb;
 
-  private String key;
+  private String batchInfoKey;
+  private String batchDataKey;
   private StorageObject storage;
   private EntityAdapterDefinition definition;
   private String whereClause;
   private String orderByClause;
-  private int nrArticlesPerBatch;
-  private int minNrArticlesInFirstBatch;
+  private int nrEntitiesToSkip;
+  private int nrEntitiesPerBatch;
+  private int minNrEntitiesInFirstBatch;
   private ProducerNode batchSubNode;
   private ProducerNode batchListSubNode;
 
   public EntityBatchingProducerNode(
-        String aKey,
+        String aBatchDataKey,
+        String aBatchInfoKey,
+        StorageObject aStorage,
+        EntityAdapterDefinition aDefinition,
+        String aWhereClause,
+        String anOrderByClause,
+        int aNrEntitiesPerBatch,
+        int aMinNrEntitiesInFirstBatch,
+        int aNrEntitiesToSkip,
+        ProducerNode aBatchSubNode) {
+    this(aBatchDataKey, aBatchInfoKey, aStorage, aDefinition, aWhereClause,
+        anOrderByClause, aNrEntitiesPerBatch, aMinNrEntitiesInFirstBatch, aNrEntitiesToSkip,
+        aBatchSubNode, null);
+  }
+
+
+  public EntityBatchingProducerNode(
+        String aBatchDataKey,
+        String aBatchInfoKey,
         StorageObject aStorage,
         EntityAdapterDefinition aDefinition,
         String aWhereClause,
         String anOrderByClause,
-        int aNrArticlesPerBatch,
-        int aMinNrArticlesInFirstBatch,
+        int aNrEntitiesPerBatch,
+        int aMinNrEntitiesInFirstBatch,
+        int aNrEntitiesToSkip,
         ProducerNode aBatchSubNode,
         ProducerNode aBatchListSubNode) {
 
     batchSubNode = aBatchSubNode;
     batchListSubNode = aBatchListSubNode;
 
-    defaultVerb = null;
     verbs = new HashMap();
 
-    key = aKey;
+    batchDataKey = aBatchDataKey;
+    batchInfoKey = aBatchInfoKey;
     storage = aStorage;
     definition = aDefinition;
     whereClause = aWhereClause;
     orderByClause = anOrderByClause;
-    nrArticlesPerBatch = aNrArticlesPerBatch;
-    minNrArticlesInFirstBatch = aMinNrArticlesInFirstBatch;
+    nrEntitiesToSkip = aNrEntitiesToSkip;
+    nrEntitiesPerBatch = aNrEntitiesPerBatch;
+    minNrEntitiesInFirstBatch = aMinNrEntitiesInFirstBatch;
   }
 
   public void produce(Map aValueMap, String aVerb, PrintWriter aLogger) throws ProducerFailure {
-    EntityBatchingProducerNodeVerb verb = (EntityBatchingProducerNodeVerb) verbs.get(aVerb);
     Iterator browser;
+    int nrEntities;
+    int nrBatchesAfterFirst;
+    int nrEntitiesInFirstBatch;
+    int nrBatchesToProcess;
+    List batchesData;
+    int i;
+    int position;
+    Map batchData;
+    String expandedWhereClause;
+    String expandedOrderByClause;
+    EntityBatchingProducerNodeVerb verb = (EntityBatchingProducerNodeVerb) verbs.get(aVerb);
+
+    List batchLocations;
+    BatchLocation location;
+
+//  ML: The order by clause should lead to a result set in _reverse order_: the first row will be
+//      the last entity presented on the last page
 
-    if (verb==null)
-      verb = defaultVerb;
 
     if (verb==null)
       throw new ProducerFailure("EntityBatchingProducerNode: unknown verb '"+aVerb+"'", null);
 
     try {
-      browser = new EntityIteratorAdapter(
-          storage,
-          ParameterExpander.expandExpression( aValueMap, verb.whereClause ),
-          ParameterExpander.expandExpression( aValueMap, verb.orderByClause ),
-          -1,
-          definition );
-
-      while (browser.hasNext()) {
-        aLogger.println( verb.whereClause + ": next... ");
-        aValueMap.put(key, browser.next());
-        super.produce(aValueMap, aVerb, aLogger);
+      expandedWhereClause = ParameterExpander.expandExpression( aValueMap, whereClause );
+      expandedOrderByClause = ParameterExpander.expandExpression( aValueMap, orderByClause );
+
+      batchesData = new Vector();
+      batchLocations = new Vector();
+
+      nrEntities = storage.getSize(expandedWhereClause)-nrEntitiesToSkip;
+      nrEntitiesInFirstBatch = nrEntities % nrEntitiesPerBatch;
+      while (nrEntitiesInFirstBatch<minNrEntitiesInFirstBatch && nrEntities-nrEntitiesInFirstBatch>=nrEntitiesPerBatch)
+        nrEntitiesInFirstBatch = nrEntitiesInFirstBatch + nrEntitiesPerBatch;
+      nrBatchesAfterFirst = (nrEntities-nrEntitiesInFirstBatch)/nrEntitiesPerBatch;
+
+      batchLocations.add(new BatchLocation(nrBatchesAfterFirst*nrEntitiesPerBatch, nrEntitiesInFirstBatch));
+      batchData = new HashMap();
+      batchData.put("identifier", "");
+      batchData.put("index", Integer.toString(1));
+      batchData.put("size", Integer.toString(nrEntitiesInFirstBatch));
+      batchesData.add(batchData);
+
+      for (i=0; i<nrBatchesAfterFirst; i++) {
+        batchLocations.add(1, new BatchLocation(i*nrEntitiesPerBatch, nrEntitiesPerBatch));
+        batchData = new HashMap();
+        batchData.put("identifier", Integer.toString(i));
+        batchData.put("index", Integer.toString(nrBatchesAfterFirst-i+1));
+        batchData.put("size", Integer.toString(nrEntitiesPerBatch));
+        batchesData.add(1, batchData);
+      }
+
+      batchData = new HashMap();
+      aValueMap.put(batchInfoKey, batchData);
+      batchData.put("all", batchesData);
+      batchData.put("first", batchesData.get(0));
+      batchData.put("last", batchesData.get(batchesData.size()-1));
+
+      if (batchListSubNode!=null) {
+        batchListSubNode.produce(aValueMap, aVerb, aLogger);
+      }
+
+      nrBatchesToProcess = nrBatchesAfterFirst+1;
+      if (verb.nrBatchesToProcess>-1 && verb.nrBatchesToProcess<nrBatchesToProcess) {
+        nrBatchesToProcess=verb.nrBatchesToProcess;
+      }
+
+      if (batchSubNode!=null) {
+        for (i=0; i<nrBatchesToProcess; i++) {
+          location = (BatchLocation) batchLocations.get(i);
+
+          batchData.put("current", batchesData.get(i));
+          if (i>0)
+            batchData.put("previous", batchesData.get(i-1));
+          else
+            batchData.put("previous", null);
+
+          if (i<batchesData.size()-1)
+            batchData.put("next", batchesData.get(i+1));
+          else
+            batchData.put("next", null);
+
+          Iterator j = new EntityIteratorAdapter(storage, expandedWhereClause, expandedOrderByClause,
+                    location.nrEntities, definition, location.nrEntities, location.firstEntity);
+          List entities = new Vector();
+
+          while (j.hasNext())
+            entities.add(0, j.next());
+
+          aValueMap.put( batchDataKey, entities );
+
+          batchSubNode.produce(aValueMap, aVerb, aLogger);
+        }
       }
     }
     catch (Throwable t) {
@@ -79,31 +171,30 @@ public class EntityBatchingProducerNode implements ProducerNode {
     }
   };
 
-  public Set buildVerbSet() {
-    Set set;
+  private class BatchLocation {
+    int nrEntities;
+    int firstEntity;
 
-    set = super.buildVerbSet();
-    set.addAll(verbs.keySet());
+    public BatchLocation(int aFirstEntity, int aNrEntities) {
+      firstEntity = aFirstEntity;
+      nrEntities = aNrEntities;
+    }
+  }
 
-    return set;
+  public Set buildVerbSet() {
+    return verbs.keySet();
   };
 
-  public void addVerb(String aVerb, String aWhereClause, String anOrderByClause) {
-    verbs.put(aVerb, new EntityBatchingProducerNodeVerb(aWhereClause, anOrderByClause));
+  public void addVerb(String aVerb, int aNrPagesToGenerate) {
+    verbs.put(aVerb, new EntityBatchingProducerNodeVerb(aNrPagesToGenerate));
   }
 
-
-// int nrPagesToGenerate
-//
-
   private class EntityBatchingProducerNodeVerb {
-    int nrPagesToGenerate;
+    int nrBatchesToProcess;
 
-    EntityBatchingProducerNodeVerb(String aWhereClause, String anOrderByClause) {
-      whereClause = aWhereClause;
-      orderByClause = anOrderByClause;
+    EntityBatchingProducerNodeVerb(int aNrBatchesToProcess) {
+      nrBatchesToProcess = aNrBatchesToProcess;
     }
   }
 }
 
-*/
\ No newline at end of file
diff --git a/source/mir/producer/EntityEnumeratingProducerNode.java b/source/mir/producer/EntityEnumeratingProducerNode.java
index e0c46427..e40a58c4 100755
--- a/source/mir/producer/EntityEnumeratingProducerNode.java
+++ b/source/mir/producer/EntityEnumeratingProducerNode.java
@@ -50,7 +50,6 @@ public class EntityEnumeratingProducerNode extends ProducerNodeDecorator {
           definition );
 
       while (browser.hasNext()) {
-        aLogger.println( verb.whereClause + ": next... ");
         aValueMap.put(key, browser.next());
         super.produce(aValueMap, aVerb, aLogger);
       }
diff --git a/source/mir/producer/EntityListProducerNode.java b/source/mir/producer/EntityListProducerNode.java
index d2aeec69..bda62413 100755
--- a/source/mir/producer/EntityListProducerNode.java
+++ b/source/mir/producer/EntityListProducerNode.java
@@ -14,10 +14,12 @@ public class EntityListProducerNode extends ProducerNodeDecorator {
   private int batchSize;
   private EntityAdapterDefinition definition;
   private StorageObject storage;
+  private int limit;
+  private int skip;
 
   public EntityListProducerNode(String aKey, StorageObject aStorage,
       EntityAdapterDefinition aDefinition, String aWhereClause, String anOrderByClause,
-      int aBatchSize, ProducerNode aSubNode) {
+      int aLimit, int aSkip, ProducerNode aSubNode) {
     super(aSubNode);
 
     storage = aStorage;
@@ -25,7 +27,8 @@ public class EntityListProducerNode extends ProducerNodeDecorator {
     key = aKey;
     whereClause = aWhereClause;
     orderByClause = anOrderByClause;
-    batchSize = aBatchSize;
+    limit = aLimit;
+    skip = aSkip;
   }
 
   public void produce(Map aValueMap, String aVerb, PrintWriter aLogger) throws ProducerFailure {
@@ -35,8 +38,10 @@ public class EntityListProducerNode extends ProducerNodeDecorator {
           storage,
           ParameterExpander.expandExpression( aValueMap, whereClause ),
           ParameterExpander.expandExpression( aValueMap, orderByClause ),
-          batchSize,
-          definition )
+          20,
+          definition,
+          limit,
+          skip )
       );
       super.produce(aValueMap, aVerb, aLogger);
     }
diff --git a/source/mircoders/entity/adapter/ContentAdapterDefinition.java b/source/mircoders/entity/adapter/ContentAdapterDefinition.java
index 6abca53f..d2e390c2 100755
--- a/source/mircoders/entity/adapter/ContentAdapterDefinition.java
+++ b/source/mircoders/entity/adapter/ContentAdapterDefinition.java
@@ -25,7 +25,9 @@ public class ContentAdapterDefinition extends EntityAdapterDefinition {
   public ContentAdapterDefinition() {
     super();
 
-    addDateField("date", "date");
+    addDBDateField("creationdate", "webdb_lastchange");
+    addDBDateField("changedate", "webdb_create");
+    addMirDateField("date", "date");
     addCalculatedField("to_topics", new ContentToTopicsField());
     addCalculatedField("to_comments", new ContentToCommentsField());
     try {
@@ -175,146 +177,6 @@ public class ContentAdapterDefinition extends EntityAdapterDefinition {
       return result;
     }
   }
-
-
-/*
-  // @todo this needs to optimized. expensive SQL
-  private SimpleHash getUploadedMediaForNewswire()
-    throws StorageObjectException, TemplateModelException
-  {
-    // fetching/setting the images
-    // return to_media_icons
-    String        tinyIcon = null, iconAlt = null;
-    MirMedia      mediaHandler = null;
-    EntityUploadedMedia uploadedMedia;
-    Entity        mediaType;
-    SimpleHash    returnHash = new SimpleHash();
-
-    EntityList upMediaEntityList =
-                    DatabaseContentToMedia.getInstance().getUploadedMedia(this);
-    if (upMediaEntityList!=null && upMediaEntityList.getCount()>=1) {
-
-      for (int n=0; n < upMediaEntityList.size();n++) {
-        uploadedMedia = (EntityUploadedMedia)upMediaEntityList.elementAt(n);
-        mediaType = uploadedMedia.getMediaType();
-        try {
-          mediaHandler = MediaHelper.getHandler( mediaType );
-        } catch (MirMediaException ex) {
-          throw new TemplateModelException(ex.toString());
-        }
-        //the "best" media type to show
-        if (mediaHandler.isVideo()) {
-          tinyIcon = MirConfig.getProp("Producer.Icon.TinyVideo");
-          iconAlt = "Video";
-          break;
-        } else if (mediaHandler.isAudio()) {
-          tinyIcon = MirConfig.getProp("Producer.Icon.TinyAudio");
-          iconAlt = "Audio";
-        } else if (tinyIcon == null && !mediaHandler.isImage()) {
-          tinyIcon = mediaHandler.getTinyIcon();
-          iconAlt = mediaHandler.getIconAlt();
-        }
-
-      }
-      //it only has image(s)
-      if (tinyIcon == null) {
-        tinyIcon = MirConfig.getProp("Producer.Icon.TinyImage");
-        iconAlt = "Image";
-      }
-    // uploadedMedia Entity list is empty.
-    // we only have text
-    } else {
-      tinyIcon = MirConfig.getProp("Producer.Icon.TinyText");
-      iconAlt = "Text";
-    }
-    returnHash.put("tiny_icon", mirconf_imageRoot+"/"+tinyIcon);
-    returnHash.put("icon_alt", iconAlt);
-    return returnHash;
-  }
-*/
-
-
 }
 
 
-//  String mirconf_extLinkName  = MirConfig.getProp("Producer.ExtLinkName");
-//  String mirconf_intLinkName  = MirConfig.getProp("Producer.IntLinkName");
-//  String mirconf_mailLinkName = MirConfig.getProp("Producer.MailLinkName");
-//  String mirconf_imageRoot    = MirConfig.getProp("Producer.ImageRoot");
-
-
-//      if (field.equals("date_formatted"))
-//      {
-//  		  if (hasValueForField("date"))
-//      	returnField = StringUtil.webdbDate2readableDate(getValue("date"));
-//  		}
-//      else if (field.equals("description_parsed"))
-//        returnField = getDescriptionParsed();
-//      else if (field.equals("description_sentence"))
-//        returnField = getDescriptionSentence();
-//      else if (field.equals("content_data_parsed"))
-//        returnField = getContentDataParsed();
-
-/*
-      else if (key.equals("to_media_icon")) {
-        try {
-          _entCache.put(key, getUploadedMediaForNewswire());
-          return (TemplateModel)_entCache.get(key);
-        }
-        catch (Exception ex) {
-          theLog.printWarning("-- getUploadedMediaForNewswire: could not fetch data " + ex.toString());
-          throw new TemplateModelException(ex.toString());
-        }
-      }
-*/
-
-/*
-  private boolean hasMedia() throws StorageObjectException
-  {
-    if (_hasMedia == null) {
-      _hasMedia =
-        new Boolean(DatabaseContentToMedia.getInstance().hasMedia(this));
-    }
-    return _hasMedia.booleanValue();
-  }
-
-  //######## @todo all of the following getBlahForContent should have
-  // and optimized version where LIMIT=1 sql for list view.
-  private EntityList getImagesForContent()
-    throws StorageObjectException, TemplateModelException
-  {
-    if (hasMedia())
-      return DatabaseContentToMedia.getInstance().getImages(this);
-    else
-      return null;
-  }
-
-  private EntityList getAudioForContent()
-    throws StorageObjectException, TemplateModelException
-  {
-    if (hasMedia())
-      return DatabaseContentToMedia.getInstance().getAudio(this) ;
-    else
-      return null;
-  }
-
-  private EntityList getVideoForContent()
-    throws StorageObjectException, TemplateModelException
-  {
-    if (hasMedia())
-      return DatabaseContentToMedia.getInstance().getVideo(this) ;
-    else
-      return null;
-  }
-
-  private EntityList getOtherMediaForContent()
-    throws StorageObjectException, TemplateModelException
-  {
-    if (hasMedia())
-      return DatabaseContentToMedia.getInstance().getOther(this);
-    else
-      return null;
-  }
-
-}
-*/
\ No newline at end of file
diff --git a/source/mircoders/global/JobQueue.java b/source/mircoders/global/JobQueue.java
index 2d578888..041094d5 100755
--- a/source/mircoders/global/JobQueue.java
+++ b/source/mircoders/global/JobQueue.java
@@ -97,6 +97,10 @@ public class JobQueue {
       return getStatus() == STATUS_PROCESSED;
     }
 
+    public boolean isPending() {
+      return getStatus() == STATUS_PENDING;
+    }
+
     private boolean setStatus(int anOldStatus, int aNewStatus) {
       synchronized(this) {
         if (status == anOldStatus) {
diff --git a/source/mircoders/global/ProducerEngine.java b/source/mircoders/global/ProducerEngine.java
index b59a523d..93216490 100755
--- a/source/mircoders/global/ProducerEngine.java
+++ b/source/mircoders/global/ProducerEngine.java
@@ -27,6 +27,7 @@ public class ProducerEngine {
   }
 
   public void addJob(String aProducerFactory, String aVerb) {
+// ML: TODO: should check if a similar job is already pending
     producerJobQueue.appendJob(new ProducerJob(aProducerFactory, aVerb));
     log.println(aProducerFactory+"."+aVerb+" added to queue");
     log.flush();
diff --git a/source/mircoders/localizer/basic/MirBasicOpenPostingLocalizer.java b/source/mircoders/localizer/basic/MirBasicOpenPostingLocalizer.java
index 805b117d..cc9627be 100755
--- a/source/mircoders/localizer/basic/MirBasicOpenPostingLocalizer.java
+++ b/source/mircoders/localizer/basic/MirBasicOpenPostingLocalizer.java
@@ -11,7 +11,7 @@ public class MirBasicOpenPostingLocalizer implements MirOpenPostingLocalizer {
   public void afterContentPosting() {
     MirGlobal.producerEngine().addJob("content", "new");
     MirGlobal.producerEngine().addJob("openposting", "new");
-    MirGlobal.producerEngine().addJob("startpage", "all");
+    MirGlobal.producerEngine().addJob("startpage", "(default)");
 
     MirGlobal.producerEngine().addJob("synchronization", "run");
   }
diff --git a/source/mircoders/localizer/basic/MirBasicProducerToolLocalizer.java b/source/mircoders/localizer/basic/MirBasicProducerToolLocalizer.java
index c1f9db22..86a7d73a 100755
--- a/source/mircoders/localizer/basic/MirBasicProducerToolLocalizer.java
+++ b/source/mircoders/localizer/basic/MirBasicProducerToolLocalizer.java
@@ -56,6 +56,14 @@ public class MirBasicProducerToolLocalizer implements MirProducerToolLocalizer {
     aValueSet.put("topics", topicList);
     aValueSet.put("imclist", entityList);
     aValueSet.put("parentlist", parentList);
+
+    Map articleTypeMap = new HashMap();
+    articleTypeMap.put("openposting", "0");
+    articleTypeMap.put("newswire", "1");
+    articleTypeMap.put("feature", "2");
+    articleTypeMap.put("topicspecial", "3");
+    articleTypeMap.put("startspecial", "4");
+    aValueSet.put("articletype", articleTypeMap);
   };
 
   public PrintWriter openWriter(String anIdentifier) throws MirLocalizerFailure {
diff --git a/source/mircoders/producer/ContentMarkingProducerNode.java b/source/mircoders/producer/ContentMarkingProducerNode.java
new file mode 100755
index 00000000..30ffa739
--- /dev/null
+++ b/source/mircoders/producer/ContentMarkingProducerNode.java
@@ -0,0 +1,48 @@
+package mircoders.producer;
+
+import java.util.*;
+import java.io.*;
+import mir.util.*;
+import mir.producer.*;
+import mir.entity.*;
+import mir.entity.adapter.*;
+import mircoders.entity.*;
+
+
+public class ContentMarkingProducerNode implements ProducerNode {
+  private String contentKey;
+
+  public ContentMarkingProducerNode(String aContentKey) {
+    contentKey = aContentKey;
+  }
+
+  public void produce(Map aValueMap, String aVerb, PrintWriter aLogger) throws ProducerFailure {
+    Object data;
+    Entity entity;
+
+    try {
+      data = ParameterExpander.findValueForKey( aValueMap, contentKey );
+
+      if (! (data instanceof EntityAdapter)) {
+        throw new ProducerFailure("ContentMarkingProducerNode: value of '"+contentKey+"' is not an EntityAdapter, but an " + data.getClass().getName(), null);
+      }
+
+      entity = ((EntityAdapter) data).getEntity();
+      if (! (entity instanceof EntityContent)) {
+        throw new ProducerFailure("ContentMarkingProducerNode: value of '"+contentKey+"' is not a content EntityAdapter, but a " + entity.getClass().getName() + " adapter", null);
+      }
+
+      ((EntityContent) entity).setProduced(true);
+    }
+    catch (Throwable t) {
+      aLogger.println("Error while marking content: " + t.getMessage());
+      t.printStackTrace(aLogger);
+
+      throw new ProducerFailure(t.getMessage(), t);
+    }
+  }
+
+  public Set buildVerbSet() {
+    return new HashSet();
+  }
+}
\ No newline at end of file
diff --git a/source/mirlocal/bolivia.indymedia.org/BoliviaProducerLocalizer.java b/source/mirlocal/bolivia.indymedia.org/BoliviaProducerLocalizer.java
index 2becc333..534d3d8c 100755
--- a/source/mirlocal/bolivia.indymedia.org/BoliviaProducerLocalizer.java
+++ b/source/mirlocal/bolivia.indymedia.org/BoliviaProducerLocalizer.java
@@ -57,32 +57,26 @@ public class BoliviaProducerLocalizer extends MirBasicProducerLocalizer {
   }
 
   protected void setupStartPageFactory(CompositeProducerFactory aStartPageFactory) {
-    ProducerNode startPageNode = null;
+/*    ProducerNode topicNode = null;
 
     try {
-      startPageNode =
-          new EntityListProducerNode("features", DatabaseContent.getInstance(), ContentAdapterDefinition.getInstance(),
-                  "is_published='1' and to_article_type = 2", "date desc, webdb_create desc", 10,
-            new EntityListProducerNode("newswire", DatabaseContent.getInstance(), ContentAdapterDefinition.getInstance(),
-                    "is_published='1' and to_article_type = 1", "date desc, webdb_create desc", 10,
-              new EntityListProducerNode("breaking", DatabaseBreaking.getInstance(), BreakingAdapterDefinition.getInstance(),
-                      "", "webdb_create desc", 5,
-                new EntityListProducerNode("languages", DatabaseLanguage.getInstance(), LanguageAdapterDefinition.getInstance(),
-                        "", "code", 10,
-                  new CompositeProducerNode( new ProducerNode[] {
-                    new EntityEnumeratingProducerNode( "language", DatabaseLanguage.getInstance(), LanguageAdapterDefinition.getInstance(), "", "",
-                      new ResourceBundleProducerNode("lang", "producer_${language.code}",
-                        new GeneratingProducerNode(
-                            "/producer/bolivia.indymedia.org/startpage.template",
-                            "${config.storageRoot}/${language.code}/index.shtml"
-                        )
-                      )
-                    )
-                  } )
+      topicNode =
+        new EntityBatchingProducerNode( "articles", "batch", DatabaseContent.getInstance(), ContentAdapterDefinition.getInstance()
+              "is_published='1' and to_article_type = 2", "date desc, webdb_create desc", 20, 10, 0,
+          new EntityListProducerNode("languages", DatabaseLanguage.getInstance(), LanguageAdapterDefinition.getInstance(),
+                  "", "code", 10, 0,
+            new EntityEnumeratingProducerNode( "language", DatabaseLanguage.getInstance(), LanguageAdapterDefinition.getInstance(), "", "",
+              new ResourceBundleProducerNode("lang", "producer_${language.code}",
+                new GeneratingProducerNode(
+                    "/producer/bolivia.indymedia.org/startpage.template",
+                    "${config.storageRoot}/${language.code}/index.shtml"
                 )
               )
             )
-          );
+          )
+        );
+      topicNode.addVerb("all", -1);
+      topicNode.addVerb("new", 3);
     }
     catch (Throwable t) {
     }
diff --git a/source/mirlocal/euskalherria.indymedia.org/EHProducerLocalizer.java b/source/mirlocal/euskalherria.indymedia.org/EHProducerLocalizer.java
index 3c7216c8..0b513b02 100755
--- a/source/mirlocal/euskalherria.indymedia.org/EHProducerLocalizer.java
+++ b/source/mirlocal/euskalherria.indymedia.org/EHProducerLocalizer.java
@@ -31,7 +31,8 @@ public class EHProducerLocalizer extends MirBasicProducerLocalizer {
                       )
                     } )
                 )
-              )
+              ),
+              new ContentMarkingProducerNode( "content")
             } )
           );
     }
@@ -39,7 +40,7 @@ public class EHProducerLocalizer extends MirBasicProducerLocalizer {
     }
 
     contentNode.addVerb( "all", "is_published='1'", "" );
-    contentNode.addVerb( "all", "is_published='1' and is_produced='f'", "" );
+    contentNode.addVerb( "new", "is_published='1' and is_produced='f'", "" );
 
     aContentFactory.addFactory( new NodedProducerFactory( contentNode ) );
 
@@ -51,17 +52,17 @@ public class EHProducerLocalizer extends MirBasicProducerLocalizer {
     try {
       startPageNode =
             new EntityListProducerNode("newswire", DatabaseContent.getInstance(), ContentAdapterDefinition.getInstance(),
-                    "is_published='1' and to_article_type = 1", "date desc, webdb_create desc", 40,
+                    "is_published='1' and to_article_type = ${articletype.newswire}", "date desc, webdb_create desc", 40, 0,
               new EntityListProducerNode("languages", DatabaseLanguage.getInstance(), LanguageAdapterDefinition.getInstance(),
-                            "", "code", 10,
+                            "", "code", 10, 0,
                 new EntityEnumeratingProducerNode( "language", DatabaseLanguage.getInstance(), LanguageAdapterDefinition.getInstance(), "", "",
                   new ResourceBundleProducerNode("lang", "bundles.producer_${language.code}",
                     new EntityListProducerNode("startspecial", DatabaseContent.getInstance(), ContentAdapterDefinition.getInstance(),
-                              "is_published='1' and to_article_type = 4", "date desc, webdb_create desc", 1,
+                              "is_published='1' and to_article_type = ${articletype.startspecial}", "date desc, webdb_create desc", 1, 0,
                       new EntityListProducerNode("features", DatabaseContent.getInstance(), ContentAdapterDefinition.getInstance(),
-                              "is_published='1' and to_article_type = 2", "date desc, webdb_create desc", 10,
+                              "is_published='1' and to_article_type = ${articletype.feature}", "date desc, webdb_create desc", 10, 0,
                         new EntityListProducerNode("breaking", DatabaseBreaking.getInstance(), BreakingAdapterDefinition.getInstance(),
-                                  "", "webdb_create desc", 5,
+                                  "", "webdb_create desc", 5, 0,
                           new GeneratingProducerNode(
                               "/producer/euskalherria.indymedia.org/startpage.template",
                               "${config.storageRoot}/${language.code}/index.shtml"
@@ -98,7 +99,7 @@ public class EHProducerLocalizer extends MirBasicProducerLocalizer {
     try {
       staticNode =
           new EntityListProducerNode("languages", DatabaseLanguage.getInstance(), LanguageAdapterDefinition.getInstance(),
-                  "", "code", 10,
+                  "", "code", 10, 0,
             new EntityEnumeratingProducerNode( "language", DatabaseLanguage.getInstance(), LanguageAdapterDefinition.getInstance(), "", "",
               new ResourceBundleProducerNode("lang", "bundles.producer_${language.code}",
                 new GeneratingProducerNode(
@@ -116,6 +117,56 @@ public class EHProducerLocalizer extends MirBasicProducerLocalizer {
   }
 
   protected void setupFactories(Map aFactoriesMap ) {
+    EntityBatchingProducerNode topicPageNode = null;
+
     super.setupFactories(aFactoriesMap);
+
+    try {
+      topicPageNode =
+        new EntityBatchingProducerNode( "articles", "batch", DatabaseContent.getInstance(), ContentAdapterDefinition.getInstance(),
+              "is_published='1' and to_article_type = ${articletype.newswire} and id in (select content_id from content_x_topic where topic_id = ${topic.id})", "date asc, webdb_create asc", 6, 5, 0,
+          new EntityListProducerNode("languages", DatabaseLanguage.getInstance(), LanguageAdapterDefinition.getInstance(),
+                  "", "code", 10, 0,
+            new EntityEnumeratingProducerNode( "language", DatabaseLanguage.getInstance(), LanguageAdapterDefinition.getInstance(), "", "",
+              new ResourceBundleProducerNode("lang", "producer_${language.code}",
+                new GeneratingProducerNode(
+                    "/producer/euskalherria.indymedia.org/topic.template",
+                    "${config.storageRoot}/${language.code}/${topic.filename}/${topic.filename}${batch.current.identifier}.shtml"
+                )
+              )
+            )
+          ),
+          new EntityListProducerNode("languages", DatabaseLanguage.getInstance(), LanguageAdapterDefinition.getInstance(),
+                  "", "code", 10, 0,
+            new EntityEnumeratingProducerNode( "language", DatabaseLanguage.getInstance(), LanguageAdapterDefinition.getInstance(), "", "",
+              new ResourceBundleProducerNode("lang", "producer_${language.code}",
+                new GeneratingProducerNode(
+                    "/producer/euskalherria.indymedia.org/topicnavigation.template",
+                    "${config.storageRoot}/${language.code}/${topic.filename}/${topic.filename}navigation.inc"
+                )
+              )
+            )
+          )
+        );
+      topicPageNode.addVerb("all", -1);
+      topicPageNode.addVerb("new", 3);
+
+      aFactoriesMap.put("topics",
+        new NodedProducerFactory(
+          new EntityEnumeratingProducerNode( "topic", DatabaseTopics.getInstance(), TopicsAdapterDefinition.getInstance(), "", "",
+            new EntityListProducerNode("topicspecial", DatabaseContent.getInstance(), ContentAdapterDefinition.getInstance(),
+                  "is_published='1' and to_article_type = ${articletype.topicspecial} and id in (select content_id from content_x_topic where topic_id = ${topic.id})", "date, webdb_create", 1, 0,
+              new EntityListProducerNode("topicfeatures", DatabaseContent.getInstance(), ContentAdapterDefinition.getInstance(),
+                    "is_published='1' and to_article_type = ${articletype.feature} and id in (select content_id from content_x_topic where topic_id = ${topic.id})", "date, webdb_create", 10, 0,
+                topicPageNode
+              )
+            )
+          )
+        )
+      );
+    }
+    catch (Throwable t) {
+    }
+
   }
 }
diff --git a/source/mirlocal/indymedia.nl/IndyNLProducerLocalizer.java b/source/mirlocal/indymedia.nl/IndyNLProducerLocalizer.java
index 0bc12e33..5ecb03ff 100755
--- a/source/mirlocal/indymedia.nl/IndyNLProducerLocalizer.java
+++ b/source/mirlocal/indymedia.nl/IndyNLProducerLocalizer.java
@@ -71,9 +71,9 @@ public class IndyNLProducerLocalizer extends MirBasicProducerLocalizer {
     try {
       startPageNode =
             new EntityListProducerNode("newswire", DatabaseContent.getInstance(), ContentAdapterDefinition.getInstance(),
-                    "is_published='1' and to_article_type = 1", "date desc, webdb_create desc", 40,
+                    "is_published='1' and to_article_type = 1", "date desc, webdb_create desc", 40, 0,
               new EntityListProducerNode("languages", DatabaseLanguage.getInstance(), LanguageAdapterDefinition.getInstance(),
-                            "", "code", 10,
+                            "", "code", 10, 0,
                     new CompositeProducerNode( new ProducerNode[] {
                         new EntityEnumeratingProducerNode( "language", DatabaseLanguage.getInstance(), LanguageAdapterDefinition.getInstance(), "", "",
                           new ResourceBundleProducerNode("lang", "bundles.producer_${language.code}",
@@ -84,9 +84,9 @@ public class IndyNLProducerLocalizer extends MirBasicProducerLocalizer {
                           )
                         ),
                         new EntityListProducerNode("features", DatabaseContent.getInstance(), ContentAdapterDefinition.getInstance(),
-                                "is_published='1' and to_article_type = 2", "date desc, webdb_create desc", 10,
+                                "is_published='1' and to_article_type = 2", "date desc, webdb_create desc", 10, 0,
                           new EntityListProducerNode("breaking", DatabaseBreaking.getInstance(), BreakingAdapterDefinition.getInstance(),
-                                    "", "webdb_create desc", 5,
+                                    "", "webdb_create desc", 5, 0,
                             new EntityEnumeratingProducerNode( "language", DatabaseLanguage.getInstance(), LanguageAdapterDefinition.getInstance(), "code='nl'", "",
                                 new AssignmentProducerNode( "staticinclude", "1",
                                   new EvaluatedAssignmentProducerNode( "topinclude", "/producer/indymedia.nl/top.template",
@@ -148,7 +148,7 @@ public class IndyNLProducerLocalizer extends MirBasicProducerLocalizer {
     try {
       staticNode =
           new EntityListProducerNode("languages", DatabaseLanguage.getInstance(), LanguageAdapterDefinition.getInstance(),
-                  "", "code", 10,
+                  "", "code", 10, 0,
               new EntityEnumeratingProducerNode( "language", DatabaseLanguage.getInstance(), LanguageAdapterDefinition.getInstance(), "", "",
                 new ResourceBundleProducerNode("lang", "bundles.producer_${language.code}",
                   new CompositeProducerNode( new ProducerNode[] {
@@ -223,11 +223,11 @@ public class IndyNLProducerLocalizer extends MirBasicProducerLocalizer {
     try {
       node =
           new EntityListProducerNode("features", DatabaseContent.getInstance(), ContentAdapterDefinition.getInstance(),
-                  "is_published='1' and to_article_type = 2", "date desc, webdb_create desc", 10,
+                  "is_published='1' and to_article_type = 2", "date desc, webdb_create desc", 10, 0,
             new EntityListProducerNode("breaking", DatabaseBreaking.getInstance(), BreakingAdapterDefinition.getInstance(),
-                      "", "webdb_create desc", 5,
+                      "", "webdb_create desc", 5, 0,
               new EntityListProducerNode("languages", DatabaseLanguage.getInstance(), LanguageAdapterDefinition.getInstance(),
-                        "", "code", 10,
+                        "", "code", 10, 0,
                 new EntityEnumeratingProducerNode( "language", DatabaseLanguage.getInstance(), LanguageAdapterDefinition.getInstance(), "", "",
                   new ResourceBundleProducerNode("lang", "bundles.producer_${language.code}",
                       new GeneratingProducerNode(
@@ -248,11 +248,11 @@ public class IndyNLProducerLocalizer extends MirBasicProducerLocalizer {
     try {
       node =
           new EntityListProducerNode("features", DatabaseContent.getInstance(), ContentAdapterDefinition.getInstance(),
-                  "is_published='1' and to_article_type = 2", "date desc, webdb_create desc", 10,
+                  "is_published='1' and to_article_type = 2", "date desc, webdb_create desc", 10, 0,
             new EntityListProducerNode("breaking", DatabaseBreaking.getInstance(), BreakingAdapterDefinition.getInstance(),
-                      "", "webdb_create desc", 5,
+                      "", "webdb_create desc", 5, 0,
               new EntityListProducerNode("languages", DatabaseLanguage.getInstance(), LanguageAdapterDefinition.getInstance(),
-                        "", "code", 10,
+                        "", "code", 10, 0,
                 new EntityEnumeratingProducerNode( "language", DatabaseLanguage.getInstance(), LanguageAdapterDefinition.getInstance(), "", "",
                   new ResourceBundleProducerNode("lang", "bundles.producer_${language.code}",
                       new GeneratingProducerNode(
-- 
2.11.0