X-Git-Url: http://erislabs.net/gitweb/?a=blobdiff_plain;f=source%2Fmircoders%2Fglobal%2FAbuse.java;h=dd54988483aa06ee56faedbd644e13ce141a33d8;hb=88455349028d7ecd46fe04ff8662b584c54c6143;hp=19475a718e2a260ecaa0e8ab49393497e9255145;hpb=54499db654731b872fed34dbf6c6fd709ada3ebe;p=mir.git diff --git a/source/mircoders/global/Abuse.java b/source/mircoders/global/Abuse.java index 19475a71..dd549884 100755 --- a/source/mircoders/global/Abuse.java +++ b/source/mircoders/global/Abuse.java @@ -30,41 +30,46 @@ package mircoders.global; -import gnu.regexp.RE; +import mir.config.MirPropertiesConfiguration; +import mir.entity.Entity; +import mir.entity.adapter.EntityAdapterModel; +import mir.log.LoggerWrapper; +import mir.session.Request; +import mir.util.DateTimeRoutines; +import mir.util.EntityUtility; +import mir.util.GeneratorFormatAdapters; +import mir.util.StringRoutines; +import mircoders.abuse.FilterEngine; +import mircoders.entity.EntityComment; +import mircoders.entity.EntityContent; +import mircoders.localizer.MirAdminInterfaceLocalizer; +import mircoders.module.ModuleComment; +import mircoders.module.ModuleContent; +import org.apache.commons.collections.ExtendedProperties; +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletResponse; +import java.io.BufferedOutputStream; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; -import java.util.Arrays; +import java.util.ArrayList; import java.util.Date; +import java.util.GregorianCalendar; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Random; -import java.util.Vector; - -import javax.servlet.http.Cookie; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import mir.entity.Entity; -import mir.log.LoggerWrapper; -import mir.session.HTTPAdapters; -import mir.session.Request; -import mir.util.DateToMapAdapter; -import mir.util.InternetFunctions; -import mir.util.StringRoutines; -import mircoders.entity.EntityComment; -import mircoders.entity.EntityContent; -import mircoders.localizer.MirAdminInterfaceLocalizer; - -import org.apache.commons.collections.ExtendedProperties; - +/** + * This class manages abuse (spam, offending material, etc.). This + * is done by using a set of filters managed by the FilterEngine class. + * Filters may be of different types (IP, throttle, regexp...), + * but are created and configured in a single user interface (web page), + * and are stored in a single database table called "filter". + */ public class Abuse { - private List filters; - private int maxIdentifier; private LoggerWrapper logger; private int logSize; private boolean logEnabled; @@ -74,19 +79,28 @@ public class Abuse { private String articleBlockAction; private String commentBlockAction; private List log; - private String configFile = MirGlobal.config().getStringWithHome("Abuse.Config"); + private File configFile = MirGlobal.config().getFile("Abuse.Config"); + private FilterEngine filterEngine; + private MirPropertiesConfiguration configuration; - private static final String IP_FILTER_TYPE="ip"; - private static final String REGEXP_FILTER_TYPE="regexp"; - private static String cookieName=MirGlobal.config().getString("Abuse.CookieName"); - private static int cookieMaxAge = 60*60*MirGlobal.config().getInt("Abuse.CookieMaxAge"); + private static String cookieName = MirGlobal.config().getString("Abuse.CookieName"); + private static int cookieMaxAge = 60 * 60 * MirGlobal.config().getInt("Abuse.CookieMaxAge"); + private EntityAdapterModel model; - public Abuse() { + public Abuse(EntityAdapterModel aModel) { logger = new LoggerWrapper("Global.Abuse"); - filters = new Vector(); - maxIdentifier = 0; - log = new Vector(); + filterEngine = new FilterEngine(aModel); + model = aModel; + + log = new ArrayList(); + + try { + configuration = MirPropertiesConfiguration.instance(); + } + catch (Throwable e) { + throw new RuntimeException("Can't get configuration: " + e.getMessage()); + } logSize = 100; logEnabled = false; @@ -99,58 +113,8 @@ public class Abuse { load(); } - public boolean checkIpFilter(String anIpAddress) { - synchronized (filters) { - Iterator i = filters.iterator(); - - while (i.hasNext()) { - Filter filter = (Filter) i.next(); - - try { - if ( (filter.getType().equals(IP_FILTER_TYPE)) && - InternetFunctions.isIpAddressInNetwork(anIpAddress, filter.getExpression())) { - logger.debug("ip match on " + filter.getExpression()); - return true; - } - } - catch (Throwable t) { - logger.warn("error while checking ip address " + anIpAddress + " over network " + filter.expression + ": " + t.getMessage()); - } - } - - return false; - } - } - - private boolean checkRegExpFilter(Entity anEntity) { - synchronized (filters) { - Iterator i = filters.iterator(); - - while (i.hasNext()) { - Filter filter = (Filter) i.next(); - - if (filter.getType().equals(REGEXP_FILTER_TYPE)) { - try { - RE regularExpression = new RE(filter.getExpression()); - - Iterator j = anEntity.getFields().iterator(); - while (j.hasNext()) { - String field = anEntity.getValue( (String) j.next()); - - if (field != null && regularExpression.isMatch(field.toLowerCase())) { - logger.debug("regexp match on " + filter.getExpression()); - return true; - } - } - } - catch (Throwable t) { - logger.warn("error while checking entity with regexp " + filter.getExpression() + ": " + t.getMessage()); - } - } - } - - return false; - } + public FilterEngine getFilterEngine() { + return filterEngine; } private void setCookie(HttpServletResponse aResponse) { @@ -160,7 +124,7 @@ public class Abuse { cookie.setMaxAge(cookieMaxAge); cookie.setPath("/"); - if (aResponse!=null) + if (aResponse != null) aResponse.addCookie(cookie); } @@ -180,71 +144,84 @@ public class Abuse { return false; } + /** Checks if there is a filter that matches a comment and takes + * appropriate action (as configured in the xxxxxaction field of + * the filter table). The actual matching is delegated to the + * FilterEngine class. + */ + public void checkComment(EntityComment aComment, Request aRequest, HttpServletResponse aResponse) { + try { + long time = System.currentTimeMillis(); - public boolean checkRequest(Request aRequest, HttpServletResponse aResponse, String anId, boolean anIsComment) { - String address = "0.0.0.0"; - String browser = "unknown"; - List cookies = null; - - HttpServletRequest request = null; - - if (aRequest instanceof HTTPAdapters.HTTPParsedRequestAdapter) { - request = ((HTTPAdapters.HTTPParsedRequestAdapter) aRequest).getRequest(); - } - else if (aRequest instanceof HTTPAdapters.HTTPRequestAdapter) { - request = ((HTTPAdapters.HTTPRequestAdapter) aRequest).getRequest(); - } - if (request!=null) { - browser = (String) request.getHeader("User-Agent"); - address = request.getRemoteAddr(); - cookies = Arrays.asList(request.getCookies()); - } + FilterEngine.Filter matchingFilter = filterEngine.testPosting(aComment, aRequest); - if (anIsComment) - logComment(address, anId , new Date(), browser); - else - logArticle(address, anId , new Date(), browser); + if (matchingFilter != null) { + logger.debug("Match for " + matchingFilter.getTag()); + matchingFilter.updateLastHit(new GregorianCalendar().getTime()); - return checkCookie(cookies) || checkIpFilter(address); - } + StringBuffer line = new StringBuffer(); - public void checkComment(EntityComment aComment, Request aRequest, HttpServletResponse aResponse) { - try { - long time = System.currentTimeMillis(); + line.append(DateTimeRoutines.advancedDateFormat( + configuration.getString("Mir.DefaultDateTimeFormat"), + (new GregorianCalendar()).getTime(), configuration.getString("Mir.DefaultTimezone"))); - MirAdminInterfaceLocalizer.MirSimpleEntityOperation operation = MirGlobal.localizer().adminInterface().simpleCommentOperationForName(commentBlockAction); + line.append(" "); + line.append(matchingFilter.getTag()); + EntityUtility.appendLineToField(aComment, "comment", line.toString()); - if (checkRequest(aRequest, aResponse, aComment.getId(), true) || checkRegExpFilter(aComment)) { - logger.debug("performing operation " + operation.getName()); - operation.perform(null, MirGlobal.localizer().dataModel().adapterModel().makeEntityAdapter("comment", aComment)); + MirGlobal.performCommentOperation(null, aComment, matchingFilter.getCommentAction()); setCookie(aResponse); + save(); + logComment(aComment, aRequest, matchingFilter.getTag()); + } + else { + logComment(aComment, aRequest); } - logger.info("checkComment: " + (System.currentTimeMillis()-time) + "ms"); + logger.debug("checkComment: " + (System.currentTimeMillis() - time) + "ms"); } catch (Throwable t) { - t.printStackTrace(logger.asPrintWriter(logger.DEBUG_MESSAGE)); - logger.error("Abuse.checkComment: " + t.toString()); + logger.error("Exception thrown while checking comment", t); } } - + /** Checks if there is a filter that matches an articleand takes + * appropriate action (as configured in the xxxxxaction field of + * the filter table). The actual matching is delegated to the + * FilterEngine class. + */ public void checkArticle(EntityContent anArticle, Request aRequest, HttpServletResponse aResponse) { try { long time = System.currentTimeMillis(); - MirAdminInterfaceLocalizer.MirSimpleEntityOperation operation = MirGlobal.localizer().adminInterface().simpleArticleOperationForName(articleBlockAction); + FilterEngine.Filter matchingFilter = filterEngine.testPosting(anArticle, aRequest); + + if (matchingFilter != null) { + logger.debug("Match for " + matchingFilter.getTag()); +// matchingFilter.updateLastHit(new GregorianCalendar().getTime()); + + StringBuffer line = new StringBuffer(); - if (checkRequest(aRequest, aResponse, anArticle.getId(), false) || checkRegExpFilter(anArticle)) { - logger.debug("performing operation " + operation.getName()); - operation.perform(null, MirGlobal.localizer().dataModel().adapterModel().makeEntityAdapter("content", anArticle)); + line.append(DateTimeRoutines.advancedDateFormat( + configuration.getString("Mir.DefaultDateTimeFormat"), + (new GregorianCalendar()).getTime(), configuration.getString("Mir.DefaultTimezone"))); + + line.append(" "); + line.append(matchingFilter.getTag()); + EntityUtility.appendLineToField(anArticle, "comment", line.toString()); + + MirGlobal.performArticleOperation(null, anArticle, matchingFilter.getArticleAction()); setCookie(aResponse); + save(); + logArticle(anArticle, aRequest, matchingFilter.getTag()); + } + else { + logArticle(anArticle, aRequest); } - logger.info("checkArticle: " + (System.currentTimeMillis()-time) + "ms"); + logger.info("checkArticle: " + (System.currentTimeMillis() - time) + "ms"); } catch (Throwable t) { - t.printStackTrace(logger.asPrintWriter(logger.DEBUG_MESSAGE)); - logger.error("Abuse.checkArticle: " + t.toString()); + logger.error("Exception thrown while checking article", t); } } @@ -253,7 +230,8 @@ public class Abuse { } public void setLogEnabled(boolean anEnabled) { - logEnabled = anEnabled; + if (!configuration.getString("Abuse.DisallowIPLogging", "0").equals("1")) + logEnabled = anEnabled; truncateLog(); } @@ -306,52 +284,90 @@ public class Abuse { commentBlockAction = anAction; } - public List getLog() { - synchronized(log) { - List result = new Vector(); + ModuleContent contentModule = new ModuleContent(); + ModuleComment commentModule = new ModuleComment(); - Iterator i = log.iterator(); - while (i.hasNext()) { - LogEntry logEntry = (LogEntry) i.next(); - Map entry = new HashMap(); - - entry.put("ip", logEntry.getIpNumber()); - entry.put("id", logEntry.getId()); - entry.put("timestamp", new DateToMapAdapter(logEntry.getTimeStamp())); - if (logEntry.getIsArticle()) - entry.put("type", "content"); - else - entry.put("type", "comment"); - entry.put("browser", logEntry.getBrowserString()); - - result.add(entry); - } + synchronized (log) { + try { + List result = new ArrayList(); - return result; + Iterator i = log.iterator(); + while (i.hasNext()) { + LogEntry logEntry = (LogEntry) i.next(); + Map entry = new HashMap(); + + entry.put("ip", logEntry.getIpNumber()); + entry.put("id", logEntry.getId()); + entry.put("timestamp", new GeneratorFormatAdapters.DateFormatAdapter(logEntry.getTimeStamp(), MirPropertiesConfiguration.instance().getString("Mir.DefaultTimezone"))); + + if (logEntry.getIsArticle()) { + entry.put("type", "content"); + entry.put("object", + model.makeEntityAdapter("content", contentModule.getById(logEntry.getId()))); + } + else { + entry.put("type", "comment"); + entry.put("object", + model.makeEntityAdapter("comment", commentModule.getById(logEntry.getId()))); + } + + entry.put("browser", logEntry.getBrowserString()); + entry.put("filtertag", logEntry.getMatchingFilterTag()); + + result.add(entry); + } + + return result; + } + catch (Throwable t) { + throw new RuntimeException(t.toString()); + } } } - public void logComment(String anIp, String anId, Date aTimeStamp, String aBrowser) { - appendLog(new LogEntry(aTimeStamp, anIp, aBrowser, anId, false)); + public void logComment(Entity aComment, Request aRequest) { + logComment(aComment, aRequest, null); } - public void logArticle(String anIp, String anId, Date aTimeStamp, String aBrowser) { - appendLog(new LogEntry(aTimeStamp, anIp, aBrowser, anId, true)); + public void logComment(Entity aComment, Request aRequest, String aMatchingFilterTag) { + String ipAddress = aRequest.getHeader("ip"); + String id = aComment.getId(); + String browser = aRequest.getHeader("User-Agent"); + + logComment(ipAddress, id, new Date(), browser, aMatchingFilterTag); + } + + public void logArticle(Entity anArticle, Request aRequest) { + logArticle(anArticle, aRequest, null); + } + + public void logArticle(Entity anArticle, Request aRequest, String aMatchingFilterTag) { + String ipAddress = aRequest.getHeader("ip"); + String id = anArticle.getId(); + String browser = aRequest.getHeader("User-Agent"); + + logArticle(ipAddress, id, new Date(), browser, aMatchingFilterTag); } - public void load() { + public void logComment(String anIp, String anId, Date aTimeStamp, String aBrowser, String aMatchingFilterTag) { + appendLog(new LogEntry(aTimeStamp, anIp, aBrowser, anId, false, aMatchingFilterTag)); + } + + public void logArticle(String anIp, String anId, Date aTimeStamp, String aBrowser, String aMatchingFilterTag) { + appendLog(new LogEntry(aTimeStamp, anIp, aBrowser, anId, true, aMatchingFilterTag)); + } + + public synchronized void load() { try { ExtendedProperties configuration = new ExtendedProperties(); try { - configuration = new ExtendedProperties(configFile); + configuration = new ExtendedProperties(configFile.getAbsolutePath()); } catch (FileNotFoundException e) { } - getFilterConfig(filters, "abuse.filter", configuration); - setOpenPostingDisabled(configuration.getString("abuse.openPostingDisabled", "0").equals("1")); setOpenPostingPassword(configuration.getString("abuse.openPostingPassword", "0").equals("1")); setCookieOnBlock(configuration.getString("abuse.cookieOnBlock", "0").equals("1")); @@ -364,46 +380,29 @@ public class Abuse { throw new RuntimeException(t.toString()); } } - public void save() { + + public synchronized void save() { try { ExtendedProperties configuration = new ExtendedProperties(); - setFilterConfig(filters, "abuse.filter", configuration); - - configuration.addProperty("abuse.openPostingDisabled", getOpenPostingDisabled()?"1":"0"); - configuration.addProperty("abuse.openPostingPassword", getOpenPostingPassword()?"1":"0"); - configuration.addProperty("abuse.cookieOnBlock", getCookieOnBlock()?"1":"0"); - configuration.addProperty("abuse.logEnabled", getLogEnabled()?"1":"0"); + configuration.addProperty("abuse.openPostingDisabled", getOpenPostingDisabled() ? "1" : "0"); + configuration.addProperty("abuse.openPostingPassword", getOpenPostingPassword() ? "1" : "0"); + configuration.addProperty("abuse.cookieOnBlock", getCookieOnBlock() ? "1" : "0"); + configuration.addProperty("abuse.logEnabled", getLogEnabled() ? "1" : "0"); configuration.addProperty("abuse.logSize", Integer.toString(getLogSize())); configuration.addProperty("abuse.articleBlockAction", getArticleBlockAction()); configuration.addProperty("abuse.commentBlockAction", getCommentBlockAction()); - configuration.save(new FileOutputStream(new File(configFile)), "Anti abuse configuration"); + configuration.save(new BufferedOutputStream(new FileOutputStream(configFile),8192), "Anti abuse configuration"); } catch (Throwable t) { throw new RuntimeException(t.toString()); } } - public List getFilterTypes() { - List result = new Vector(); - - Map entry = new HashMap(); - entry.put("resource", "ip"); - entry.put("id", IP_FILTER_TYPE); - result.add(entry); - - entry = new HashMap(); - entry.put("resource", "regexp"); - entry.put("id", REGEXP_FILTER_TYPE); - result.add(entry); - - return result; - } - public List getArticleActions() { try { - List result = new Vector(); + List result = new ArrayList(); Iterator i = MirGlobal.localizer().adminInterface().simpleArticleOperations().iterator(); while (i.hasNext()) { @@ -426,7 +425,7 @@ public class Abuse { public List getCommentActions() { try { - List result = new Vector(); + List result = new ArrayList(); Iterator i = MirGlobal.localizer().adminInterface().simpleCommentOperations().iterator(); while (i.hasNext()) { @@ -447,163 +446,23 @@ public class Abuse { } } - public List getFilters() { - return getFiltersAsMaps(filters); - } - - public void addFilter(String aType, String anExpression) { - addFilter(filters, aType, anExpression); - } - - public void setFilter(String anIdentifier, String aType, String anExpression) { - setFilter(filters, anIdentifier, aType, anExpression); - } - - public void deleteFilter(String anIdentifier) { - deleteFilter(filters, anIdentifier); - } - - public void validateIpFilter(String anIdentifier, String anArticleAction, String aCommentAction) throws Exception { - } - - private List getFiltersAsMaps(List aFilters) { - synchronized(aFilters) { - List result = new Vector(); - - Iterator i = aFilters.iterator(); - while (i.hasNext()) { - Filter filter = (Filter) i.next(); - Map map = new HashMap(); - - map.put("id", filter.getId()); - map.put("expression", filter.getExpression()); - map.put("type", filter.getType()); - - result.add(map); - } - return result; - } - } - - private void addFilter(List aFilters, String aType, String anExpression) { - Filter filter = new Filter(); - - filter.setId(generateId()); - filter.setExpression(anExpression); - filter.setType(aType); - - synchronized (aFilters) { - aFilters.add(filter); - } - } - - private void setFilter(List aFilters, String anIdentifier, String aType, String anExpression) { - synchronized (aFilters) { - Filter filter = findFilter(aFilters, anIdentifier); - - if (filter!=null) { - filter.setExpression(anExpression); - filter.setType(aType); - } - } - } - - private Filter findFilter(List aFilters, String anIdentifier) { - synchronized (aFilters) { - Iterator i = aFilters.iterator(); - while (i.hasNext()) { - Filter filter = (Filter) i.next(); - - if (filter.getId().equals(anIdentifier)) { - return filter; - } - } - } - - return null; - } - - private void deleteFilter(List aFilters, String anIdentifier) { - synchronized (aFilters) { - Filter filter = findFilter(aFilters, anIdentifier); - - if (filter!=null) { - aFilters.remove(filter); - } - } + private String escapeConfigListEntry(String aFilterPart) { + return StringRoutines.replaceStringCharacters(aFilterPart, + new char[] {'\\', ':'}, + new String[] {"\\\\", "\\:"}); } - private String generateId() { - synchronized(this) { - maxIdentifier = maxIdentifier+1; - - return Integer.toString(maxIdentifier); - } + private String escapeFilterPart(String aFilterPart) { + return StringRoutines.replaceStringCharacters(aFilterPart, + new char[] {'\\', '\n', '\r', '\t', ' '}, + new String[] {"\\\\", "\\n", "\\r", "\\t", "\\ "}); } - private static class Filter { - private String identifier; - private String expression; - private String type; - - public Filter() { - expression=""; - type=""; - identifier=""; - } - - public String getId() { - return identifier; - } - - public void setId(String anId) { - identifier = anId; - } - - public String getExpression() { - return expression; - } - - public void setExpression(String anExpression) { - expression = anExpression; - } - - public String getType() { - return type; - } - - public void setType(String aType) { - type = aType; - } - } - - private void setFilterConfig(List aFilters, String aConfigKey, ExtendedProperties aConfiguration) { - synchronized(aFilters) { - Iterator i = aFilters.iterator(); - - while (i.hasNext()) { - Filter filter = (Filter) i.next(); - - aConfiguration.addProperty(aConfigKey, filter.getType()+":"+filter.getExpression()); - } - } - } - - private void getFilterConfig(List aFilters, String aConfigKey, ExtendedProperties aConfiguration) { - synchronized(aFilters) { - aFilters.clear(); - - Iterator i = Arrays.asList(aConfiguration.getStringArray(aConfigKey)).iterator(); - - while (i.hasNext()) { - String filter = (String) i.next(); - List parts = StringRoutines.separateString(filter, ":"); - - if (parts.size()==2) { - addFilter( (String) parts.get(0), (String) parts.get(1)); - } - } - } + private String deescapeFilterPart(String aFilterPart) { + return StringRoutines.replaceEscapedStringCharacters(aFilterPart, + '\\', + new char[] {'\\', ':', 'n', 'r', 't', ' '}, + new String[] {"\\", ":", "\n", "\r", "\t", " "}); } private static class LogEntry { @@ -612,13 +471,19 @@ public class Abuse { private String id; private Date timeStamp; private boolean isArticle; + private String matchingFilterTag; - public LogEntry(Date aTimeStamp, String anIpNumber, String aBrowserString, String anId, boolean anIsArticle) { + public LogEntry(Date aTimeStamp, String anIpNumber, String aBrowserString, String anId, boolean anIsArticle, String aMatchingFilterTag) { ipNumber = anIpNumber; browserString = aBrowserString; id = anId; isArticle = anIsArticle; - timeStamp=aTimeStamp; + timeStamp = aTimeStamp; + matchingFilterTag = aMatchingFilterTag; + } + + public LogEntry(Date aTimeStamp, String anIpNumber, String aBrowserString, String anId, boolean anIsArticle) { + this(aTimeStamp, anIpNumber, aBrowserString, anId, anIsArticle, null); } public String getIpNumber() { @@ -633,6 +498,10 @@ public class Abuse { return id; } + public String getMatchingFilterTag() { + return matchingFilterTag; + } + public Date getTimeStamp() { return timeStamp; } @@ -643,24 +512,23 @@ public class Abuse { } private void truncateLog() { - synchronized(log) { + synchronized (log) { if (!logEnabled) log.clear(); else { - while (log.size()>0 && log.size()>logSize) { - log.remove(0); + while (log.size() > 0 && log.size() > logSize) { + log.remove(log.size()-1); } } } - }; + } private void appendLog(LogEntry anEntry) { synchronized (log) { if (logEnabled) { - log.add(anEntry); + log.add(0, anEntry); truncateLog(); } } } - -} \ No newline at end of file +}