utopiah
/
misc
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

refreshing the logs scripts

Browse Source
master
Fabien Benetou 14 years ago
parent 598a903aac
commit 96ca35b735
3 changed files with 25 additions and 7 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 16
      shell_scripts/browser_queries
  2. 10
      shell_scripts/logs_anchors_generation
  3. 6
      shell_scripts/logs_links_extraction

16
shell_scripts/browser_queries
Unescape View File

@ -17,6 +17,7 @@
# Note that the name might be misleading since currently only Firefox SQLite format is supported # Note that the name might be misleading since currently only Firefox SQLite format is supported
ANHOURAGO=`date +%s --date "now 1 hour ago"` ANHOURAGO=`date +%s --date "now 1 hour ago"`
ADAYAGO=`date +%s --date "now 1 day ago"`
AWEEKAGO=`date +%s --date "now 1 week ago"` AWEEKAGO=`date +%s --date "now 1 week ago"`
DB=~/.mozilla/firefox/*.default/places.sqlite DB=~/.mozilla/firefox/*.default/places.sqlite
@ -27,7 +28,8 @@ echo -e "\ntop 10 visited URL in general"
echo "this week" echo "this week"
#sqlite3 -list -line $DB "SELECT url,title,visit_date as moment FROM moz_historyvisits, moz_places WHERE moz_historyvisits.place_id = moz_places.id and visit_date>($AWEEKAGO*1000000)" #sqlite3 -list -line $DB "SELECT url,title,visit_date as moment FROM moz_historyvisits, moz_places WHERE moz_historyvisits.place_id = moz_places.id and visit_date>($AWEEKAGO*1000000)"
echo "this hour" echo "this hour"
sqlite3 -list -line $DB "SELECT url,title,visit_date as moment FROM moz_historyvisits, moz_places WHERE moz_historyvisits.place_id = moz_places.id and visit_date>($ANHOURAGO*1000000)" #sqlite3 -list -line $DB "SELECT url,title,visit_date as moment FROM moz_historyvisits, moz_places WHERE moz_historyvisits.place_id = moz_places.id and visit_date>($ANHOURAGO*1000000)"
#sqlite3 -list -line $DB "SELECT url,title,visit_date as moment FROM moz_historyvisits, moz_places WHERE moz_historyvisits.place_id = moz_places.id and visit_date>($ADAYAGO*1000000)"
URL=$1 URL=$1
echo -e "\nhow many time URL X has been visited this week" echo -e "\nhow many time URL X has been visited this week"
@ -44,12 +46,12 @@ echo -e "\nURLs visited between starting URL and ending URL"
URLSLISTFILE=~/.urlstocheck URLSLISTFILE=~/.urlstocheck
SCRIPTURL=`grep '^\$ScriptUrl' ~/www/mirrors/fabien/local/config.php | grep -v 127 | sed -e "s/.*'\(.*\)';/\1/"` #SCRIPTURL=`grep '^\$ScriptUrl' ~/www/mirrors/fabien/local/config.php | grep -v 127 | sed -e "s/.*'\(.*\)';/\1/"`
echo "" > $URLSLISTFILE #echo "" > $URLSLISTFILE
ls ~/www/mirrors/fabien/wiki.d/ | grep -v "del-" | tr "." "/" | sed -e "s,^,$SCRIPTURL\/," >> $URLSLISTFILE #ls ~/www/mirrors/fabien/wiki.d/ | grep -v "del-" | tr "." "/" | sed -e "s,^,$SCRIPTURL\/," >> $URLSLISTFILE
while read line; do #while read line; do
sqlite3 -column $DB "SELECT count(*),url FROM moz_historyvisits, moz_places WHERE moz_historyvisits.place_id = moz_places.id and url LIKE '$line'" | grep "0" && echo -e "\t$line" # sqlite3 -column $DB "SELECT count(*),url FROM moz_historyvisits, moz_places WHERE moz_historyvisits.place_id = moz_places.id and url LIKE '$line'" | grep "0" && echo -e "\t$line"
done < $URLSLISTFILE #done < $URLSLISTFILE
echo -e "\nwhich URLs from this list have never been visited this week" echo -e "\nwhich URLs from this list have never been visited this week"
echo -e "\nwhich URLs from this list have never been visited in general" echo -e "\nwhich URLs from this list have never been visited in general"
# e.g. all wiki pages, all page from a wiki group (e.g. memory recall) # e.g. all wiki pages, all page from a wiki group (e.g. memory recall)

10
shell_scripts/logs_anchors_generation
Unescape View File

@ -0,0 +1,10 @@
#! /usr/bin/env bash
# TODO
# escape HTML characters
# slice by month as very long logs are problematic
# generalize for non supybot log
LOG_SOURCE=/home/utopiah/web/thelab/stigmergylive/logs/ChannelLogger/freenode/##pim/##pim.log
HTML_DEST=/home/utopiah/web/thelab/stigmergylive/logs/ChannelLogger/freenode/##pim/anchoredlog.html
echo "<html>" > $HTML_DEST && cat $LOG_SOURCE | sed "s/</[/" | sed "s/>/]/" | sed "s/\(.*\) \([*[].*\)/<a name=\"\1\"><\/a><a href=\"\/at\/\1\">\1<\/a> \2/" | sed "s/^/<p>/" | sed "s/$/<\/p>/" >> $HTML_DEST && echo "</html>" >> $HTML_DEST

6
shell_scripts/logs_links_extraction
Unescape View File

@ -0,0 +1,6 @@
#! /usr/bin/env bash
SOURCE_LOG=/home/utopiah/web/thelab/stigmergylive/logs/ChannelLogger/freenode/##pim/##pim.log
DEST_HTML=/home/utopiah/web/pim/links_fresh.html
echo "<html>" > $DEST_HTML && cat $SOURCE_LOG | grep http: | sed -e "s/.*\(http\S*\).*/<a href=\"\1\">\1<\/a><br \/>/" | grep -v seedea.org > $DEST_HTML && echo "</html>" >> $DEST_HTML
Write Preview
Loading…
Cancel
Save