From: ianb Date: Sun, 18 Jan 2009 03:00:22 +0000 (+0000) Subject: scripts/mir-setup: new scripts: X-Git-Url: http://erislabs.net/gitweb/?p=mir.git;a=commitdiff_plain;h=46afc417dc18313424f13fa1b46d30e28d588fd4 scripts/mir-setup: new scripts: mir-prepare: prepare a fresh (debian) machine to run mir mir-choose-java: configure java implementation for mir to use mir-empty-db: empty a mir db to prepare for a database restore strip-ips-from-apache-error-log: anonymise apache error logs, installed by mir-prepare --- diff --git a/scripts/mir-setup/mir-choose-java b/scripts/mir-setup/mir-choose-java new file mode 100755 index 00000000..b6899b65 --- /dev/null +++ b/scripts/mir-setup/mir-choose-java @@ -0,0 +1,142 @@ +#!/bin/sh +# ianb@erislabs.net 20081202 +# configure a mir system for sun or gcj java +# checkpoint code taken from zak's mir-setup + +set -e + +# NAME is used when called from mir-prepare +NAME="" + +CUR_CHECKPOINT=0 +checkpoint() +{ + CUR_CHECKPOINT=$(($CUR_CHECKPOINT + 1)) + if [ -n "$START_CHECKPOINT" ]; then + if [ "$CUR_CHECKPOINT" -lt "$START_CHECKPOINT" ]; then + echo "${NAME}Skipping checkpoint $CUR_CHECKPOINT: $@" + unset RUNNING + else + echo "${NAME}Running from checkpoint $CUR_CHECKPOINT: $@" + RUNNING=1 + fi + else + echo "${NAME}Checkpoint $CUR_CHECKPOINT: $@" + RUNNING=1 + fi +} + +optdie() +{ + echo >&2 "$@"; + echo >&2 "Try $PROG --help"; + exit 1; +} + +usage() { + echo >&2 "Usage: $PROG [options] sun|gcj" + echo >&2 " Configures java implementation to use with mir" + echo >&2 " -d|--debug Debugging output, including 'set -x' shell trace" + echo >&2 " -p|--checkpoint Run from specified checkpoint after failed run" + echo >&2 " sun Use Sun Java (non-free)" + echo >&2 " gcj Use the GNU Java Compiler" +} + +if [ $# -lt 1 ] +then + usage + exit 1 +fi + +while [ $# != 0 ]; do + case "$1" in + -d|--debug) set -x; DEBUG=1 ;; + -p|--checkpoint) shift; START_CHECKPOINT="$1" ;; + --subprocess) NAME="[mir-choose-java] " ;; + -h|-?|--help) usage; exit 0 ;; + -*) optdie "$PROG: unknown option $1" ;; + sun) JAVA=sun ; break;; + gcj) JAVA=gcj ; break;; + *) optdie "$PROG: unexpected argument '$1'" ;; + esac + shift +done + +if [ $JAVA = sun ] +then + JAVANAME=java-6-sun + PACKAGES="sun-java6-jdk sun-java6-jre" + RTPATH=/usr/lib/jvm/java-6-sun/jre/lib/rt.jar +else + JAVANAME=java-gcj + PACKAGES=gcj + RTPATH=/usr/lib/jvm/java-1.5.0-gcj-4.3-1.5.0.0/jre/lib/rt.jar +fi + +JAVAHOME="/usr/lib/jvm/$JAVANAME" + +if [ $JAVA = sun ] +then + checkpoint "Adding non-free archive for sun java" + if [ -n "$RUNNING" ] + then + perl -pi.bak -e \ + 'if(/^deb/ && !/security\.debian\.org/ && !/non-free/) { s/(main)/$1 contrib non-free/; }' \ + /etc/apt/sources.list + apt-get update + fi +else + # so checkpoint numbers don't differ + checkpoint "Skipping non-free archive configuration - gcj is in main" +fi + +checkpoint "Installing packages" +if [ -n "$RUNNING" ] +then + apt-get install $PACKAGES + echo "You probably want to do 'apt-get clean'" +fi + +checkpoint "Selecting JDK" +if [ -n "$RUNNING" ] +then + update-java-alternatives --set $JAVANAME || true +fi + +checkpoint "Configuring apache jakarta JVM" +if [ -n "$RUNNING" ] +then + if grep -q '^workers.java_home' /etc/libapache2-mod-jk/workers.properties + then + perl -pi.bak -e 's,workers.java_home.*,workers.java_home='"$JAVAHOME"',;' \ + /etc/libapache2-mod-jk/workers.properties + fi +fi + +checkpoint "Configuring tomcat JVM" +if [ -n "$RUNNING" ] +then + TOMCATDEFAULTS=/etc/default/tomcat5.5 + if grep -q '^#\?JAVA_HOME' $TOMCATDEFAULTS + then + perl -pi.bak -e 's,^\#?JAVA_HOME.*,JAVA_HOME='"$JAVAHOME"',;' $TOMCATDEFAULTS + else + echo "JAVA_HOME=$JAVAHOME" >> $TOMCATDEFAULTS + fi +fi + +checkpoint "Configuring /etc/mir-setup/lib/rt.jar symlink" +if [ -n "$RUNNING" ] +then + mkdir -p /etc/mir-setup/lib + ln -sf $RTPATH /etc/mir-setup/lib/rt.jar +fi + +checkpoint "Restarting tomcat and apache" +if [ -n "$RUNNING" ] +then + invoke-rc.d tomcat5.5 force-reload + invoke-rc.d apache2 force-reload +fi + +checkpoint "Finished!" diff --git a/scripts/mir-setup/mir-empty-db b/scripts/mir-setup/mir-empty-db new file mode 100644 index 00000000..d5da8cf6 --- /dev/null +++ b/scripts/mir-setup/mir-empty-db @@ -0,0 +1,50 @@ +#!/bin/sh +# ianb@erislabs.net 20080212 +# empty out a mir db ready to reload a database dump + +# uploaded_media should be last because other tables depend on it +TABLES="active_article_classification active_groups + active_multimedia_file active_publishbuffer + active_ratings_system active_usertable active_webcast + active_webcast_groups active_webcast_groups_appear + active_weblink article_type audio breaking comment + comment_status comment_x_media content content_x_media + content_x_topic features_category features_feature_categories + features_features features_imc_site_info features_language + features_site_languages features_tmp_features_count filter + filter_group images img_color img_format img_layout img_type + language media_folder media_type messages other_media rights + topic video webdb_users + uploaded_media" + +SEQUENCES="category_id_seq features_id_seq imc_site_info_id_seq + groupid webcastid weblinkid publishbufferid + article_classificationid webcast_groupsid + webcast_groups_appearid usertableid multimedia_fileid + ratings_systemid media_folder_id_seq media_type_id_seq + language_id_seq comment_status_id_seq webdb_users_id_seq + article_type_id_seq topic_id_seq comment_id_seq + media_id_seq breaking_id_seq messages_id_seq filter_id_seq + filter_group_id_seq" + + +if [ $# -lt 1 ] +then + echo "usage: $0 [dbname]" 1>&2 + echo " deletes mir db ready for a database restore" 1>&2 + echo " USE WITH CARE - will destroy your mir database" 1>&2 + exit 1 +fi + +( + echo "DROP LANGUAGE plpgsql;" + echo "DROP FUNCTION plpgsql_call_handler ( ) ;" + for i in $TABLES + do + echo "DROP TABLE $i;" + done + for i in $SEQUENCES + do + echo "DROP SEQUENCE $i;" + done +) | sudo -u postgres psql "$1" diff --git a/scripts/mir-setup/mir-prepare b/scripts/mir-setup/mir-prepare new file mode 100755 index 00000000..cf0f9161 --- /dev/null +++ b/scripts/mir-setup/mir-prepare @@ -0,0 +1,180 @@ +#!/bin/sh +# ianb@erislabs.net 20081122 +# checkpoint code taken from zak's mir-setup +# $Id: mir-prepare,v 1.1.2.1 2009/01/18 03:00:22 ianb Exp $ + +set -e + +CONFDIR="/etc/mir-setup" +PROG="`basename $0`" + +PACKAGES="openssh-server pwgen apache2 ant tomcat5.5 tomcat5.5-admin imagemagick postgresql libjmagick6-java libapache2-mod-jk procmail cvs rsync sudo wget" + +CUR_CHECKPOINT=0 +checkpoint() +{ + CUR_CHECKPOINT=$(($CUR_CHECKPOINT + 1)) + if [ -n "$START_CHECKPOINT" ]; then + if [ "$CUR_CHECKPOINT" -lt "$START_CHECKPOINT" ]; then + echo "Skipping checkpoint $CUR_CHECKPOINT: $@" + unset RUNNING + else + echo "Running from checkpoint $CUR_CHECKPOINT: $@" + RUNNING=1 + fi + else + echo "Checkpoint $CUR_CHECKPOINT: $@" + RUNNING=1 + fi +} + +die() +{ + echo >&2 "$@"; + exit 1; +} + +optdie() +{ + echo >&2 "$@"; + echo >&2 "Try $PROG --help"; + exit 1; +} + +usage() { + echo >&2 "Usage: $PROG options" + echo >&2 " Prepares system for a mir install" + echo >&2 " Run before mir-setup" + echo >&2 " -d|--debug Debugging output, including 'set -x' shell trace" + echo >&2 " -p|--checkpoint Run from specified checkpoint after failed run" + echo >&2 " -j|--java sun|gcj Select Java implementation. Default: sun" +} + +# default +JAVA=sun +while [ $# != 0 ]; do + case "$1" in + -d|--debug) set -x; DEBUG=1 ;; + -p|--checkpoint) shift; START_CHECKPOINT="$1" ;; + -j|--java) shift; JAVA="$1" ;; + -h|-?|--help) usage; exit 0 ;; + -*) optdie "$PROG: unknown option $1" ;; + *) optdie "$PROG: unexpected argument '$1'" ;; + esac + shift +done + +checkpoint "Installing mir-setup" +if [ -n "$RUNNING" ] +then + mkdir -p /etc/mir-setup /usr/local/share/mir-setup + cp mir-setup mir-choose-java /usr/local/sbin + cp munge_config_file.pl /usr/local/share/mir-setup + cp conf/* /etc/mir-setup +fi + +checkpoint "Installing packages" +if [ -n "$RUNNING" ] +then + apt-get install $PACKAGES + echo "You probably want to do 'apt-get clean'" +fi + +checkpoint "Selecting $JAVA java with mir-choose-java" +if [ -n "$RUNNING" ] +then + mir-choose-java --subprocess $JAVA +fi + +checkpoint "Enabling apache modules" +if [ -n "$RUNNING" ] +then + a2enmod headers ssl include deflate jk rewrite +fi + +checkpoint "Anonymise apache error log" +if [ -n "$RUNNING" ] +then + if [ ! -f /etc/apache2/conf.d/mir-anon-error-log ] + then + echo 'ErrorLog "| /usr/local/bin/strip-ips-from-apache-error-log /var/log/apache2/error.log"' > /etc/apache2/conf.d/mir-anon-error-log + fi + cp strip-ips-from-apache-error-log /usr/local/bin +fi + +checkpoint "Configure 'anon' log format" +if [ -n "$RUNNING" ] +then + if [ ! -f /etc/apache2/conf.d/mir-anon-access-log ] + then + echo 'LogFormat "noip %l %u %t \"%r\" %>s %b" anon' > /etc/apache2/conf.d/mir-anon-access-log + fi +fi + +checkpoint "Creating directory for mir sites in apache" +if [ -n "$RUNNING" ] +then + if [ ! -d /etc/apache2/mir-sites ] + then + mkdir /etc/apache2/mir-sites + fi +fi + +checkpoint "Enabling mir sites in apache" +if [ -n "$RUNNING" ] +then + if [ ! -f /etc/apache2/sites-available/mir-sites ] + then + echo "Include /etc/apache2/mir-sites/*.conf" > /etc/apache2/sites-available/mir-sites + a2ensite mir-sites + fi +fi + +checkpoint "Configuring apache jakarta workers" +if [ -n "$RUNNING" ] +then + if [ ! -f /etc/apache2/conf.d/mir-jk ] + then + echo "JkWorkersFile /etc/libapache2-mod-jk/workers.properties" > /etc/apache2/conf.d/mir-jk + fi +fi + +checkpoint "Disabling tomcat security (yes, this is bad)" +if [ -n "$RUNNING" ] +then + if ! grep -q "^TOMCAT5_SECURITY=no" /etc/default/tomcat5.5 + then + echo "TOMCAT5_SECURITY=no" >> /etc/default/tomcat5.5 + fi +fi + +checkpoint "Configuring password for tomcat manager" +if [ -n "$RUNNING" ] +then + if ! grep -q 'user username="mir-setup"' /etc/tomcat5.5/tomcat-users.xml + then + PASSWORD=$(pwgen -n 10 1) + ROLE=' ' + USERNAME=' ' + perl -pi.bak -e 's/(mir-setup:)[^@]+/$1'"$PASSWORD"'/;' /etc/mir-setup/tomcat-manager.conf + perl -pi.bak -e 's/()/$1\n'"$ROLE"'\n'"$USERNAME"'\n/;' /etc/tomcat5.5/tomcat-users.xml + fi +fi + +checkpoint "Creating directory for SSL certs" +if [ -n "$RUNNING" ] +then + if [ ! -d /etc/apache2/ssl ] + then + mkdir /etc/apache2/ssl + fi +fi + +checkpoint "Restarting apache and tomcat" +if [ -n "$RUNNING" ] +then + invoke-rc.d tomcat5.5 force-reload + invoke-rc.d apache2 force-reload +fi + +checkpoint "All done!" diff --git a/scripts/mir-setup/strip-ips-from-apache-error-log b/scripts/mir-setup/strip-ips-from-apache-error-log new file mode 100755 index 00000000..f605a40b --- /dev/null +++ b/scripts/mir-setup/strip-ips-from-apache-error-log @@ -0,0 +1,29 @@ +#!/usr/bin/perl -w + +# From http://docs.indymedia.org/view/Sysadmin/ApacheLogsWithoutIPs +# 10 April 2005 + +# Cleaned up -zak 2005-04-10 + +use strict; + +# check we're called with exactly one argument, our output file name +if ( @ARGV != 1 ) {die "I need exactly one argument ";} + +my $out=$ARGV[0]; + +open (OUT,">> $out") or die "Failed to open $out for writing: $! \n"; + +# turn on buffer autoflush so tail -f error.log works +# (rather than waiting till file handle is closed +my $outf = select(OUT); +$|=1; +select($outf); + + +# Strip IP's and replace with 0.0.0.0 +while () { + s/\[client.*\]/\[client 0.0.0.0\]/; + print OUT; +} +close (OUT);