Reorganization step 2.

5 files changed, 112 insertions, 0 deletions

diff --git a/woip/sh/livesearch b/woip/sh/livesearch
new file mode 100755
index 0000000..9f4cc6f
--- /dev/null
+++ b/woip/sh/livesearch
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+../c/livesearch $1.processed $1.locate.db $1.locate.prefixdb $1.blocks.db
\ No newline at end of file
diff --git a/woip/sh/process b/woip/sh/process
new file mode 100755
index 0000000..9ca4476
--- /dev/null
+++ b/woip/sh/process
@@ -0,0 +1,48 @@
+#!/bin/zsh
+
+if [ ! -n "$1" ]; then
+  echo "process <file>"
+  exit -1
+else
+  file=$1
+fi
+
+ofile="$file.processed"
+if [ ! -f $ofile ]; then
+  echo "Creating processed version of $file..."
+  bzcat $file | ruby ../rb/xmlprocess.rb | bzip2 > $ofile
+else
+  echo "$ofile already exists"
+fi
+
+ifile="$file.index.txt"
+if [ ! -f $ifile ]; then
+  echo "Creating index..."
+  ruby ../rb/index.rb $ofile > $ifile
+else
+  echo "$ifile already exists"
+fi
+
+sfile="$file.locate.db"
+if [ ! -f $sfile ]; then
+  echo "Creating locate index..."
+  cat $ifile | LC_ALL=C /usr/libexec/locate.mklocatedb > $sfile
+else
+  echo "$sfile already exists"
+fi
+
+spfile="$file.locate.prefixdb"
+if [ ! -f $spfile ]; then
+  echo "Creating locate prefix index..."
+  ../c/lsearcher -f $sfile -c $spfile -n
+else
+  echo "$spfile already exists"
+fi
+
+bfile="$file.blocks.db"
+if [ ! -f $bfile ]; then
+  echo "Creating block index"
+  ../c/bzipreader -f $ofile -l | awk '{print $2;}' | ../c/blocks $bfile
+else
+  echo "$bfile already exists"
+fi
diff --git a/woip/sh/scan b/woip/sh/scan
new file mode 100755
index 0000000..3a49d44
--- /dev/null
+++ b/woip/sh/scan
@@ -0,0 +1,28 @@
+#!/bin/zsh
+# print bz2 debug info for given file in roundabout fashion
+
+file=$1
+ftmp="/tmp/$$.tmp"
+otmp="/tmp/$$.o.tmp"
+
+if [ ! -n "$1" ]; then
+  echo "scan <file>" >/dev/stdout
+  exit -1
+fi
+
+if [ -n "$2" ]; then
+  arg=$2
+else
+  arg="2>/dev/null"
+fi
+
+../c/bzipreader -f $file -l > $ftmp
+
+I=-1; while [ $((I++)) -lt 10 ];      
+do
+  offset=`cat $ftmp | grep -E "^$I" | awk '{print $2;}'`
+  echo -n "$I\t$offset\t"
+  eval "../c/bzipreader -f $file -n $I -x $arg > $otmp"
+  cat $otmp | head | tr "\n" "\\" | head -c 30
+  echo "" `cat $otmp | wc -c`
+done
diff --git a/woip/sh/server b/woip/sh/server
new file mode 100755
index 0000000..231d077
--- /dev/null
+++ b/woip/sh/server
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+ruby -r ../rb/server.rb -e 'WPServer.start_on(9090)' -- $1
diff --git a/woip/sh/test b/woip/sh/test
new file mode 100755
index 0000000..f3313ba
--- /dev/null
+++ b/woip/sh/test
@@ -0,0 +1,30 @@
+#!/bin/zsh
+
+file="../ga.wp.txt.bz2"
+bz="../c/bzipreader -f $file"
+pfile="../gawiki-latest-pages-articles.xml.bz2.processed"
+
+pushd ../c; make clean; make; popd
+
+function fail() {
+ echo "FAIL:" $* >/dev/stdout
+ exit -1
+}
+
+if ! (eval "$bz -x -n 1 2>/dev/null" | grep -q Daoine); then
+  fail decompression 
+fi
+
+if ! (eval "$bz -x -b 0" | grep -q Esperanto); then
+  fail decompression with 0 offset
+fi
+
+last=`eval "$bz -l | tail -1 | awk '{print \\$2;}'"`
+if ! (eval "$bz -x -b $last" | grep -q Nikon); then
+  fail decompression with $last offset
+fi
+
+mv $pfile ../ga.tmp
+rm ../gawiki-*
+mv ../ga.tmp $pfile
+../sh/process `echo "$pfile" | sed -E 's/\.proc.+//'`