From cef68a8f6eacf75443cb040f3df3f6c09284679b Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Mon, 23 Sep 2013 13:42:39 +0200 Subject: [PATCH] Don't keep the collections open --- database.go | 75 ++++++++++++++++++++++++++++++---------------------- mapreduce.go | 57 ++++++++++++++++++--------------------- 2 files changed, 69 insertions(+), 63 deletions(-) diff --git a/database.go b/database.go index e22693f..deadfcf 100644 --- a/database.go +++ b/database.go @@ -42,11 +42,6 @@ type News struct { type DB struct { session *mgo.Session - books *mgo.Collection - user *mgo.Collection - news *mgo.Collection - stats *mgo.Collection - mr *MR } func initDB() *DB { @@ -56,13 +51,6 @@ func initDB() *DB { if err != nil { panic(err) } - - database := d.session.DB(DB_NAME) - d.books = database.C(BOOKS_COLL) - d.user = database.C(USERS_COLL) - d.news = database.C(NEWS_COLL) - d.stats = database.C(STATS_COLL) - d.mr = NewMR(database) return d } @@ -78,12 +66,14 @@ func md5Pass(pass string) []byte { func (d *DB) SetPassword(user string, pass string) error { hash := md5Pass(pass) - return d.user.Update(bson.M{"user": user}, bson.M{"$set": bson.M{"pass": hash}}) + userColl := d.session.DB(DB_NAME).C(USERS_COLL) + return userColl.Update(bson.M{"user": user}, bson.M{"$set": bson.M{"pass": hash}}) } func (d *DB) UserValid(user string, pass string) bool { hash := md5Pass(pass) - n, err := d.user.Find(bson.M{"user": user, "pass": hash}).Count() + userColl := d.session.DB(DB_NAME).C(USERS_COLL) + n, err := userColl.Find(bson.M{"user": user, "pass": hash}).Count() if err != nil { return false } @@ -95,7 +85,8 @@ func (d *DB) UserRole(user string) string { Role string } res := result{} - err := d.user.Find(bson.M{"user": user}).One(&res) + userColl := d.session.DB(DB_NAME).C(USERS_COLL) + err := userColl.Find(bson.M{"user": user}).One(&res) if err != nil { return "" } @@ -106,7 +97,8 @@ func (d *DB) AddNews(text string) error { var news News news.Text = text news.Date = time.Now() - return d.news.Insert(news) + newsColl := d.session.DB(DB_NAME).C(NEWS_COLL) + return newsColl.Insert(news) } func (d *DB) GetNews(num int, days int) (news []News, err error) { @@ -116,25 +108,30 @@ func (d *DB) GetNews(num int, days int) (news []News, err error) { date := time.Now().Add(duration) query = bson.M{"date": bson.M{"$gt": date}} } - q := d.news.Find(query).Sort("-date").Limit(num) + newsColl := d.session.DB(DB_NAME).C(NEWS_COLL) + q := newsColl.Find(query).Sort("-date").Limit(num) err = q.All(&news) return } func (d *DB) InsertStats(stats interface{}) error { - return d.stats.Insert(stats) + statsColl := d.session.DB(DB_NAME).C(STATS_COLL) + return statsColl.Insert(stats) } func (d *DB) InsertBook(book interface{}) error { - return d.books.Insert(book) + booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL) + return booksColl.Insert(book) } func (d *DB) RemoveBook(id bson.ObjectId) error { - return d.books.Remove(bson.M{"_id": id}) + booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL) + return booksColl.Remove(bson.M{"_id": id}) } func (d *DB) UpdateBook(id bson.ObjectId, data interface{}) error { - return d.books.Update(bson.M{"_id": id}, bson.M{"$set": data}) + booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL) + return booksColl.Update(bson.M{"_id": id}, bson.M{"$set": data}) } /* optional parameters: length and start index @@ -149,7 +146,8 @@ func (d *DB) GetBooks(query bson.M, r ...int) (books []Book, num int, err error) start = r[1] } } - q := d.books.Find(query).Sort("-_id") + booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL) + q := booksColl.Find(query).Sort("-_id") num, err = q.Count() if err != nil { return @@ -171,14 +169,17 @@ func (d *DB) GetBooks(query bson.M, r ...int) (books []Book, num int, err error) /* Get the most visited books */ func (d *DB) GetVisitedBooks(num int) (books []Book, err error) { - bookId, err := d.mr.GetMostVisited(num, d.stats) + statsColl := d.session.DB(DB_NAME).C(STATS_COLL) + mr := NewMR(d.session.DB(DB_NAME)) + bookId, err := mr.GetMostVisited(num, statsColl) if err != nil { return nil, err } books = make([]Book, num) for i, id := range bookId { - d.books.Find(bson.M{"_id": id}).One(&books[i]) + booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL) + booksColl.Find(bson.M{"_id": id}).One(&books[i]) books[i].Id = bson.ObjectId(books[i].Id).Hex() } return @@ -187,14 +188,17 @@ func (d *DB) GetVisitedBooks(num int) (books []Book, err error) { /* Get the most downloaded books */ func (d *DB) GetDownloadedBooks(num int) (books []Book, err error) { - bookId, err := d.mr.GetMostDownloaded(num, d.stats) + statsColl := d.session.DB(DB_NAME).C(STATS_COLL) + mr := NewMR(d.session.DB(DB_NAME)) + bookId, err := mr.GetMostDownloaded(num, statsColl) if err != nil { return nil, err } books = make([]Book, num) for i, id := range bookId { - d.books.Find(bson.M{"_id": id}).One(&books[i]) + booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL) + booksColl.Find(bson.M{"_id": id}).One(&books[i]) books[i].Id = bson.ObjectId(books[i].Id).Hex() } return @@ -210,7 +214,8 @@ func (d *DB) GetNewBooks(r ...int) (books []Book, num int, err error) { func (d *DB) BookActive(id bson.ObjectId) bool { var book Book - err := d.books.Find(bson.M{"_id": id}).One(&book) + booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL) + err := booksColl.Find(bson.M{"_id": id}).One(&book) if err != nil { return false } @@ -222,7 +227,9 @@ func (d *DB) GetFS(prefix string) *mgo.GridFS { } func (d *DB) GetTags(numTags int) ([]string, error) { - return d.mr.GetTags(numTags, d.books) + booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL) + mr := NewMR(d.session.DB(DB_NAME)) + return mr.GetTags(numTags, booksColl) } type Visits struct { @@ -231,13 +238,19 @@ type Visits struct { } func (d *DB) GetHourVisits(start time.Time) ([]Visits, error) { - return d.mr.GetHourVisits(start, d.stats) + statsColl := d.session.DB(DB_NAME).C(STATS_COLL) + mr := NewMR(d.session.DB(DB_NAME)) + return mr.GetHourVisits(start, statsColl) } func (d *DB) GetDayVisits(start time.Time) ([]Visits, error) { - return d.mr.GetDayVisits(start, d.stats) + statsColl := d.session.DB(DB_NAME).C(STATS_COLL) + mr := NewMR(d.session.DB(DB_NAME)) + return mr.GetDayVisits(start, statsColl) } func (d *DB) GetMonthVisits(start time.Time) ([]Visits, error) { - return d.mr.GetMonthVisits(start, d.stats) + statsColl := d.session.DB(DB_NAME).C(STATS_COLL) + mr := NewMR(d.session.DB(DB_NAME)) + return mr.GetMonthVisits(start, statsColl) } diff --git a/mapreduce.go b/mapreduce.go index dbadd19..4f4b55c 100644 --- a/mapreduce.go +++ b/mapreduce.go @@ -7,30 +7,12 @@ import ( ) type MR struct { - meta *mgo.Collection - tags *mgo.Collection - visited *mgo.Collection - downloaded *mgo.Collection - hourly_raw *mgo.Collection - daily_raw *mgo.Collection - monthly_raw *mgo.Collection - hourly *mgo.Collection - daily *mgo.Collection - monthly *mgo.Collection + database *mgo.Database } func NewMR(database *mgo.Database) *MR { m := new(MR) - m.meta = database.C(META_COLL) - m.tags = database.C(TAGS_COLL) - m.visited = database.C(VISITED_COLL) - m.downloaded = database.C(DOWNLOADED_COLL) - m.hourly_raw = database.C(HOURLY_VISITS_COLL + "_raw") - m.daily_raw = database.C(DAILY_VISITS_COLL + "_raw") - m.monthly_raw = database.C(MONTHLY_VISITS_COLL + "_raw") - m.hourly = database.C(HOURLY_VISITS_COLL) - m.daily = database.C(DAILY_VISITS_COLL) - m.monthly = database.C(MONTHLY_VISITS_COLL) + m.database = database return m } @@ -56,7 +38,8 @@ func (m *MR) GetTags(numTags int, booksColl *mgo.Collection) ([]string, error) { var result []struct { Tag string "_id" } - err := m.tags.Find(nil).Sort("-value").Limit(numTags).All(&result) + tagsColl := m.database.C(TAGS_COLL) + err := tagsColl.Find(nil).Sort("-value").Limit(numTags).All(&result) if err != nil { return nil, err } @@ -88,7 +71,8 @@ func (m *MR) GetMostVisited(num int, statsColl *mgo.Collection) ([]bson.ObjectId var result []struct { Book bson.ObjectId "_id" } - err := m.visited.Find(nil).Sort("-value").Limit(num).All(&result) + visitedColl := m.database.C(VISITED_COLL) + err := visitedColl.Find(nil).Sort("-value").Limit(num).All(&result) if err != nil { return nil, err } @@ -120,7 +104,8 @@ func (m *MR) GetMostDownloaded(num int, statsColl *mgo.Collection) ([]bson.Objec var result []struct { Book bson.ObjectId "_id" } - err := m.downloaded.Find(nil).Sort("-value").Limit(num).All(&result) + downloadedColl := m.database.C(DOWNLOADED_COLL) + err := downloadedColl.Find(nil).Sort("-value").Limit(num).All(&result) if err != nil { return nil, err } @@ -157,14 +142,16 @@ func (m *MR) GetHourVisits(start time.Time, statsColl *mgo.Collection) ([]Visits emit(this['_id']['date'], 1); }` mr2.Reduce = reduce - err = m.update(&mr2, bson.M{}, m.hourly_raw, HOURLY_VISITS_COLL) + hourly_raw := m.database.C(HOURLY_VISITS_COLL + "_raw") + err = m.update(&mr2, bson.M{}, hourly_raw, HOURLY_VISITS_COLL) if err != nil { return nil, err } } var result []Visits - err := m.hourly.Find(nil).All(&result) + hourlyColl := m.database.C(HOURLY_VISITS_COLL) + err := hourlyColl.Find(nil).All(&result) return result, err } @@ -192,14 +179,16 @@ func (m *MR) GetDayVisits(start time.Time, statsColl *mgo.Collection) ([]Visits, emit(this['_id']['date'], 1); }` mr2.Reduce = reduce - err = m.update(&mr2, bson.M{}, m.daily_raw, DAILY_VISITS_COLL) + daily_raw := m.database.C(DAILY_VISITS_COLL + "_raw") + err = m.update(&mr2, bson.M{}, daily_raw, DAILY_VISITS_COLL) if err != nil { return nil, err } } var result []Visits - err := m.daily.Find(nil).All(&result) + dailyColl := m.database.C(DAILY_VISITS_COLL) + err := dailyColl.Find(nil).All(&result) return result, err } @@ -226,19 +215,22 @@ func (m *MR) GetMonthVisits(start time.Time, statsColl *mgo.Collection) ([]Visit emit(this['_id']['date'], 1); }` mr2.Reduce = reduce - err = m.update(&mr2, bson.M{}, m.monthly_raw, MONTHLY_VISITS_COLL) + monthly_raw := m.database.C(MONTHLY_VISITS_COLL + "_raw") + err = m.update(&mr2, bson.M{}, monthly_raw, MONTHLY_VISITS_COLL) if err != nil { return nil, err } } var result []Visits - err := m.monthly.Find(nil).All(&result) + monthlyColl := m.database.C(MONTHLY_VISITS_COLL) + err := monthlyColl.Find(nil).All(&result) return result, err } func (m *MR) update(mr *mgo.MapReduce, query bson.M, queryColl *mgo.Collection, storeColl string) error { - _, err := m.meta.RemoveAll(bson.M{"type": storeColl}) + metaColl := m.database.C(META_COLL) + _, err := metaColl.RemoveAll(bson.M{"type": storeColl}) if err != nil { return err } @@ -249,14 +241,15 @@ func (m *MR) update(mr *mgo.MapReduce, query bson.M, queryColl *mgo.Collection, return err } - return m.meta.Insert(bson.M{"type": storeColl}) + return metaColl.Insert(bson.M{"type": storeColl}) } func (m *MR) isOutdated(coll string, minutes float64) bool { var result struct { Id bson.ObjectId `bson:"_id"` } - err := m.meta.Find(bson.M{"type": coll}).One(&result) + metaColl := m.database.C(META_COLL) + err := metaColl.Find(bson.M{"type": coll}).One(&result) if err != nil { return true }