From a4ccd41d0577c9d58520f1f1c4c301416c3090e9 Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Sat, 1 Jun 2013 02:34:11 +0200 Subject: [PATCH] Use an strcut for mapreduce stuff --- config.go | 30 +++++++------- database.go | 85 ++-------------------------------------- mapreduce.go | 108 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 95 deletions(-) create mode 100644 mapreduce.go diff --git a/config.go b/config.go index c144607..465e659 100644 --- a/config.go +++ b/config.go @@ -3,21 +3,23 @@ package main const ( PORT = "8080" - DB_IP = "127.0.0.1" - DB_NAME = "trantor" - META_COLL = "meta" - BOOKS_COLL = "books" - TAGS_COLL = "tags" - USERS_COLL = "users" - STATS_COLL = "statistics" - FS_BOOKS = "fs_books" - FS_IMGS = "fs_imgs" + DB_IP = "127.0.0.1" + DB_NAME = "trantor" + META_COLL = "meta" + BOOKS_COLL = "books" + TAGS_COLL = "tags" + DAILY_VISITS_COLL = "visits.daily" + USERS_COLL = "users" + STATS_COLL = "statistics" + FS_BOOKS = "fs_books" + FS_IMGS = "fs_imgs" - PASS_SALT = "ImperialLibSalt" - MINUTES_UPDATE_TAGS = 10 - TAGS_DISPLAY = 50 - SEARCH_ITEMS_PAGE = 20 - NEW_ITEMS_PAGE = 50 + PASS_SALT = "ImperialLibSalt" + MINUTES_UPDATE_TAGS = 10 + MINUTES_UPDATE_DAILY = 60 * 12 + TAGS_DISPLAY = 50 + SEARCH_ITEMS_PAGE = 20 + NEW_ITEMS_PAGE = 50 TEMPLATE_PATH = "templates/" CSS_PATH = "css/" diff --git a/database.go b/database.go index ea03fd0..9a8fa98 100644 --- a/database.go +++ b/database.go @@ -7,10 +7,6 @@ import ( "time" ) -const ( - META_TYPE_TAGS = "tags updated" -) - var db *DB type Book struct { @@ -40,11 +36,10 @@ type Book struct { type DB struct { session *mgo.Session - meta *mgo.Collection books *mgo.Collection - tags *mgo.Collection user *mgo.Collection stats *mgo.Collection + mr *MR } func initDB() *DB { @@ -56,11 +51,10 @@ func initDB() *DB { } database := d.session.DB(DB_NAME) - d.meta = database.C(META_COLL) d.books = database.C(BOOKS_COLL) - d.tags = database.C(TAGS_COLL) d.user = database.C(USERS_COLL) d.stats = database.C(STATS_COLL) + d.mr = NewMR(database) return d } @@ -188,65 +182,8 @@ func (d *DB) GetFS(prefix string) *mgo.GridFS { return d.session.DB(DB_NAME).GridFS(prefix) } -func (d *DB) areTagsOutdated() bool { - var result struct { - Id bson.ObjectId `bson:"_id"` - } - err := d.meta.Find(bson.M{"type": META_TYPE_TAGS}).One(&result) - if err != nil { - return true - } - - lastUpdate := result.Id.Time() - return time.Since(lastUpdate).Minutes() > MINUTES_UPDATE_TAGS -} - -func (d *DB) updateTags() error { - _, err := d.meta.RemoveAll(bson.M{"type": META_TYPE_TAGS}) - if err != nil { - return err - } - - var mr mgo.MapReduce - mr.Map = `function() { - if (this.active) { - this.subject.forEach(function(s) { emit(s, 1); }); - } - }` - mr.Reduce = `function(tag, vals) { - var count = 0; - vals.forEach(function() { count += 1; }); - return count; - }` - mr.Out = bson.M{"replace": TAGS_COLL} - _, err = d.books.Find(bson.M{"active": true}).MapReduce(&mr, nil) - if err != nil { - return err - } - - return d.meta.Insert(bson.M{"type": META_TYPE_TAGS}) -} - func (d *DB) GetTags(numTags int) ([]string, error) { - if d.areTagsOutdated() { - err := d.updateTags() - if err != nil { - return nil, err - } - } - - var result []struct { - Tag string "_id" - } - err := d.tags.Find(nil).Sort("-value").Limit(numTags).All(&result) - if err != nil { - return nil, err - } - tags := make([]string, len(result)) - for i, r := range result { - tags[i] = r.Tag - } - return tags, nil + return d.mr.GetTags(numTags, d.books) } type Visits struct { @@ -255,19 +192,5 @@ type Visits struct { } func (d *DB) GetDayVisits(start time.Time) ([]Visits, error) { - var mr mgo.MapReduce - mr.Map = `function() { - var day = Date.UTC(this.date.getFullYear(), - this.date.getMonth(), - this.date.getDate()); - emit(day, 1); - }` - mr.Reduce = `function(date, vals) { - var count = 0; - vals.forEach(function(v) { count += v; }); - return count; - }` - var result []Visits - _, err := d.stats.Find(bson.M{"date": bson.M{"$gte": start}}).MapReduce(&mr, &result) - return result, err + return d.mr.GetDayVisits(start, d.stats) } diff --git a/mapreduce.go b/mapreduce.go new file mode 100644 index 0000000..d46c8c3 --- /dev/null +++ b/mapreduce.go @@ -0,0 +1,108 @@ +package main + +import ( + "labix.org/v2/mgo" + "labix.org/v2/mgo/bson" + "time" +) + +type MR struct { + meta *mgo.Collection + tags *mgo.Collection + daily *mgo.Collection +} + +func NewMR(database *mgo.Database) *MR { + m := new(MR) + m.meta = database.C(META_COLL) + m.tags = database.C(TAGS_COLL) + m.daily = database.C(DAILY_VISITS_COLL) + return m +} + +func (m *MR) GetTags(numTags int, booksColl *mgo.Collection) ([]string, error) { + if m.isOutdated(TAGS_COLL, MINUTES_UPDATE_TAGS) { + var mr mgo.MapReduce + mr.Map = `function() { + if (this.subject) { + this.subject.forEach(function(s) { emit(s, 1); }); + } + }` + mr.Reduce = `function(tag, vals) { + var count = 0; + vals.forEach(function() { count += 1; }); + return count; + }` + err := m.update(&mr, bson.M{"active": true}, booksColl, TAGS_COLL) + if err != nil { + return nil, err + } + } + + var result []struct { + Tag string "_id" + } + err := m.tags.Find(nil).Sort("-value").Limit(numTags).All(&result) + if err != nil { + return nil, err + } + + tags := make([]string, len(result)) + for i, r := range result { + tags[i] = r.Tag + } + return tags, nil +} + +func (m *MR) GetDayVisits(start time.Time, statsColl *mgo.Collection) ([]Visits, error) { + if m.isOutdated(DAILY_VISITS_COLL, MINUTES_UPDATE_DAILY) { + var mr mgo.MapReduce + mr.Map = `function() { + var day = Date.UTC(this.date.getFullYear(), + this.date.getMonth(), + this.date.getDate()); + emit(day, 1); + }` + mr.Reduce = `function(date, vals) { + var count = 0; + vals.forEach(function(v) { count += v; }); + return count; + }` + err := m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, DAILY_VISITS_COLL) + if err != nil { + return nil, err + } + } + + var result []Visits + err := m.daily.Find(nil).All(&result) + return result, err +} + +func (m *MR) update(mr *mgo.MapReduce, query bson.M, queryColl *mgo.Collection, storeColl string) error { + _, err := m.meta.RemoveAll(bson.M{"type": storeColl}) + if err != nil { + return err + } + + mr.Out = bson.M{"replace": storeColl} + _, err = queryColl.Find(query).MapReduce(mr, nil) + if err != nil { + return err + } + + return m.meta.Insert(bson.M{"type": storeColl}) +} + +func (m *MR) isOutdated(coll string, minutes float64) bool { + var result struct { + Id bson.ObjectId `bson:"_id"` + } + err := m.meta.Find(bson.M{"type": coll}).One(&result) + if err != nil { + return true + } + + lastUpdate := result.Id.Time() + return time.Since(lastUpdate).Minutes() > minutes +}