archives/trantor
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use an strcut for mapreduce stuff

Browse source
This commit is contained in:
Las Zenow 2013-06-01 02:34:11 +02:00
parent d8e46419b1
commit a4ccd41d05
3 changed files with 128 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
config.go
View file

@ -3,21 +3,23 @@ package main
const (
PORT = "8080"
DB_IP = "127.0.0.1"
DB_NAME = "trantor"
META_COLL = "meta"
BOOKS_COLL = "books"
TAGS_COLL = "tags"
USERS_COLL = "users"
STATS_COLL = "statistics"
FS_BOOKS = "fs_books"
FS_IMGS = "fs_imgs"
DB_IP = "127.0.0.1"
DB_NAME = "trantor"
META_COLL = "meta"
BOOKS_COLL = "books"
TAGS_COLL = "tags"
DAILY_VISITS_COLL = "visits.daily"
USERS_COLL = "users"
STATS_COLL = "statistics"
FS_BOOKS = "fs_books"
FS_IMGS = "fs_imgs"
PASS_SALT = "ImperialLibSalt"
MINUTES_UPDATE_TAGS = 10
TAGS_DISPLAY = 50
SEARCH_ITEMS_PAGE = 20
NEW_ITEMS_PAGE = 50
PASS_SALT = "ImperialLibSalt"
MINUTES_UPDATE_TAGS = 10
MINUTES_UPDATE_DAILY = 60 * 12
TAGS_DISPLAY = 50
SEARCH_ITEMS_PAGE = 20
NEW_ITEMS_PAGE = 50
TEMPLATE_PATH = "templates/"
CSS_PATH = "css/"

85
database.go
View file

@ -7,10 +7,6 @@ import (
"time"
)
const (
META_TYPE_TAGS = "tags updated"
)
var db *DB
type Book struct {
@ -40,11 +36,10 @@ type Book struct {
type DB struct {
session *mgo.Session
meta *mgo.Collection
books *mgo.Collection
tags *mgo.Collection
user *mgo.Collection
stats *mgo.Collection
mr *MR
}
func initDB() *DB {
@ -56,11 +51,10 @@ func initDB() *DB {
}
database := d.session.DB(DB_NAME)
d.meta = database.C(META_COLL)
d.books = database.C(BOOKS_COLL)
d.tags = database.C(TAGS_COLL)
d.user = database.C(USERS_COLL)
d.stats = database.C(STATS_COLL)
d.mr = NewMR(database)
return d
}
@ -188,65 +182,8 @@ func (d *DB) GetFS(prefix string) *mgo.GridFS {
return d.session.DB(DB_NAME).GridFS(prefix)
}
func (d *DB) areTagsOutdated() bool {
var result struct {
Id bson.ObjectId `bson:"_id"`
}
err := d.meta.Find(bson.M{"type": META_TYPE_TAGS}).One(&result)
if err != nil {
return true
}
lastUpdate := result.Id.Time()
return time.Since(lastUpdate).Minutes() > MINUTES_UPDATE_TAGS
}
func (d *DB) updateTags() error {
_, err := d.meta.RemoveAll(bson.M{"type": META_TYPE_TAGS})
if err != nil {
return err
}
var mr mgo.MapReduce
mr.Map = `function() {
if (this.active) {
this.subject.forEach(function(s) { emit(s, 1); });
}
}`
mr.Reduce = `function(tag, vals) {
var count = 0;
vals.forEach(function() { count += 1; });
return count;
}`
mr.Out = bson.M{"replace": TAGS_COLL}
_, err = d.books.Find(bson.M{"active": true}).MapReduce(&mr, nil)
if err != nil {
return err
}
return d.meta.Insert(bson.M{"type": META_TYPE_TAGS})
}
func (d *DB) GetTags(numTags int) ([]string, error) {
if d.areTagsOutdated() {
err := d.updateTags()
if err != nil {
return nil, err
}
}
var result []struct {
Tag string "_id"
}
err := d.tags.Find(nil).Sort("-value").Limit(numTags).All(&result)
if err != nil {
return nil, err
}
tags := make([]string, len(result))
for i, r := range result {
tags[i] = r.Tag
}
return tags, nil
return d.mr.GetTags(numTags, d.books)
}
type Visits struct {
@ -255,19 +192,5 @@ type Visits struct {
}
func (d *DB) GetDayVisits(start time.Time) ([]Visits, error) {
var mr mgo.MapReduce
mr.Map = `function() {
var day = Date.UTC(this.date.getFullYear(),
this.date.getMonth(),
this.date.getDate());
emit(day, 1);
}`
mr.Reduce = `function(date, vals) {
var count = 0;
vals.forEach(function(v) { count += v; });
return count;
}`
var result []Visits
_, err := d.stats.Find(bson.M{"date": bson.M{"$gte": start}}).MapReduce(&mr, &result)
return result, err
return d.mr.GetDayVisits(start, d.stats)
}

108
mapreduce.go Normal file
View file

@ -0,0 +1,108 @@
package main
import (
"labix.org/v2/mgo"
"labix.org/v2/mgo/bson"
"time"
)
type MR struct {
meta *mgo.Collection
tags *mgo.Collection
daily *mgo.Collection
}
func NewMR(database *mgo.Database) *MR {
m := new(MR)
m.meta = database.C(META_COLL)
m.tags = database.C(TAGS_COLL)
m.daily = database.C(DAILY_VISITS_COLL)
return m
}
func (m *MR) GetTags(numTags int, booksColl *mgo.Collection) ([]string, error) {
if m.isOutdated(TAGS_COLL, MINUTES_UPDATE_TAGS) {
var mr mgo.MapReduce
mr.Map = `function() {
if (this.subject) {
this.subject.forEach(function(s) { emit(s, 1); });
}
}`
mr.Reduce = `function(tag, vals) {
var count = 0;
vals.forEach(function() { count += 1; });
return count;
}`
err := m.update(&mr, bson.M{"active": true}, booksColl, TAGS_COLL)
if err != nil {
return nil, err
}
}
var result []struct {
Tag string "_id"
}
err := m.tags.Find(nil).Sort("-value").Limit(numTags).All(&result)
if err != nil {
return nil, err
}
tags := make([]string, len(result))
for i, r := range result {
tags[i] = r.Tag
}
return tags, nil
}
func (m *MR) GetDayVisits(start time.Time, statsColl *mgo.Collection) ([]Visits, error) {
if m.isOutdated(DAILY_VISITS_COLL, MINUTES_UPDATE_DAILY) {
var mr mgo.MapReduce
mr.Map = `function() {
var day = Date.UTC(this.date.getFullYear(),
this.date.getMonth(),
this.date.getDate());
emit(day, 1);
}`
mr.Reduce = `function(date, vals) {
var count = 0;
vals.forEach(function(v) { count += v; });
return count;
}`
err := m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, DAILY_VISITS_COLL)
if err != nil {
return nil, err
}
}
var result []Visits
err := m.daily.Find(nil).All(&result)
return result, err
}
func (m *MR) update(mr *mgo.MapReduce, query bson.M, queryColl *mgo.Collection, storeColl string) error {
_, err := m.meta.RemoveAll(bson.M{"type": storeColl})
if err != nil {
return err
}
mr.Out = bson.M{"replace": storeColl}
_, err = queryColl.Find(query).MapReduce(mr, nil)
if err != nil {
return err
}
return m.meta.Insert(bson.M{"type": storeColl})
}
func (m *MR) isOutdated(coll string, minutes float64) bool {
var result struct {
Id bson.ObjectId `bson:"_id"`
}
err := m.meta.Find(bson.M{"type": coll}).One(&result)
if err != nil {
return true
}
lastUpdate := result.Id.Time()
return time.Since(lastUpdate).Minutes() > minutes
}