#! /usr/bin/env bash

WIKIPATH=.

if [ $# -gt 0 ]
then
	WIKIPATH=$1
fi

# lots of cool usage to explore
#	automatic category creation
#	annotated images

# to facilitate the creation .stopwords run few times
#	head ../pub/keywords_distribution/* | sort -n | tail -30

for P in `ls $WIKIPATH -IPmWiki.* -I*RecentChanges -Itotalcounter.stat -I*,del-*`; do grep ^text= $P | sed "s/^text=//" | sed "s/%0a/\\n/g" | sed "s/^(:.*:)//g" | sed "s/^>>.*//g" | sed "s/http.*|//g" | sed "s/[^a-zA-Z]/\\n/g" | sort | uniq -i -c | grep -i -v -f $WIKIPATH/.stopwords | sort -r -n > $WIKIPATH/../pub/keywords_distribution/$P.txt; done