archives/trantor
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Don't keep the collections open

Browse source
This commit is contained in:
Las Zenow 2013-09-23 13:42:39 +02:00
parent f5ae093e02
commit cef68a8f6e
2 changed files with 69 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
database.go
View file

@ -42,11 +42,6 @@ type News struct {
type DB struct {
session *mgo.Session
books *mgo.Collection
user *mgo.Collection
news *mgo.Collection
stats *mgo.Collection
mr *MR
}
func initDB() *DB {
@ -56,13 +51,6 @@ func initDB() *DB {
if err != nil {
panic(err)
}
database := d.session.DB(DB_NAME)
d.books = database.C(BOOKS_COLL)
d.user = database.C(USERS_COLL)
d.news = database.C(NEWS_COLL)
d.stats = database.C(STATS_COLL)
d.mr = NewMR(database)
return d
}
@ -78,12 +66,14 @@ func md5Pass(pass string) []byte {
func (d *DB) SetPassword(user string, pass string) error {
hash := md5Pass(pass)
return d.user.Update(bson.M{"user": user}, bson.M{"$set": bson.M{"pass": hash}})
userColl := d.session.DB(DB_NAME).C(USERS_COLL)
return userColl.Update(bson.M{"user": user}, bson.M{"$set": bson.M{"pass": hash}})
}
func (d *DB) UserValid(user string, pass string) bool {
hash := md5Pass(pass)
n, err := d.user.Find(bson.M{"user": user, "pass": hash}).Count()
userColl := d.session.DB(DB_NAME).C(USERS_COLL)
n, err := userColl.Find(bson.M{"user": user, "pass": hash}).Count()
if err != nil {
return false
}
@ -95,7 +85,8 @@ func (d *DB) UserRole(user string) string {
Role string
}
res := result{}
err := d.user.Find(bson.M{"user": user}).One(&res)
userColl := d.session.DB(DB_NAME).C(USERS_COLL)
err := userColl.Find(bson.M{"user": user}).One(&res)
if err != nil {
return ""
}
@ -106,7 +97,8 @@ func (d *DB) AddNews(text string) error {
var news News
news.Text = text
news.Date = time.Now()
return d.news.Insert(news)
newsColl := d.session.DB(DB_NAME).C(NEWS_COLL)
return newsColl.Insert(news)
}
func (d *DB) GetNews(num int, days int) (news []News, err error) {
@ -116,25 +108,30 @@ func (d *DB) GetNews(num int, days int) (news []News, err error) {
date := time.Now().Add(duration)
query = bson.M{"date": bson.M{"$gt": date}}
}
q := d.news.Find(query).Sort("-date").Limit(num)
newsColl := d.session.DB(DB_NAME).C(NEWS_COLL)
q := newsColl.Find(query).Sort("-date").Limit(num)
err = q.All(&news)
return
}
func (d *DB) InsertStats(stats interface{}) error {
return d.stats.Insert(stats)
statsColl := d.session.DB(DB_NAME).C(STATS_COLL)
return statsColl.Insert(stats)
}
func (d *DB) InsertBook(book interface{}) error {
return d.books.Insert(book)
booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL)
return booksColl.Insert(book)
}
func (d *DB) RemoveBook(id bson.ObjectId) error {
return d.books.Remove(bson.M{"_id": id})
booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL)
return booksColl.Remove(bson.M{"_id": id})
}
func (d *DB) UpdateBook(id bson.ObjectId, data interface{}) error {
return d.books.Update(bson.M{"_id": id}, bson.M{"$set": data})
booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL)
return booksColl.Update(bson.M{"_id": id}, bson.M{"$set": data})
}
/* optional parameters: length and start index
@ -149,7 +146,8 @@ func (d *DB) GetBooks(query bson.M, r ...int) (books []Book, num int, err error)
start = r[1]
}
}
q := d.books.Find(query).Sort("-_id")
booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL)
q := booksColl.Find(query).Sort("-_id")
num, err = q.Count()
if err != nil {
return
@ -171,14 +169,17 @@ func (d *DB) GetBooks(query bson.M, r ...int) (books []Book, num int, err error)
/* Get the most visited books
*/
func (d *DB) GetVisitedBooks(num int) (books []Book, err error) {
bookId, err := d.mr.GetMostVisited(num, d.stats)
statsColl := d.session.DB(DB_NAME).C(STATS_COLL)
mr := NewMR(d.session.DB(DB_NAME))
bookId, err := mr.GetMostVisited(num, statsColl)
if err != nil {
return nil, err
}
books = make([]Book, num)
for i, id := range bookId {
d.books.Find(bson.M{"_id": id}).One(&books[i])
booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL)
booksColl.Find(bson.M{"_id": id}).One(&books[i])
books[i].Id = bson.ObjectId(books[i].Id).Hex()
}
return
@ -187,14 +188,17 @@ func (d *DB) GetVisitedBooks(num int) (books []Book, err error) {
/* Get the most downloaded books
*/
func (d *DB) GetDownloadedBooks(num int) (books []Book, err error) {
bookId, err := d.mr.GetMostDownloaded(num, d.stats)
statsColl := d.session.DB(DB_NAME).C(STATS_COLL)
mr := NewMR(d.session.DB(DB_NAME))
bookId, err := mr.GetMostDownloaded(num, statsColl)
if err != nil {
return nil, err
}
books = make([]Book, num)
for i, id := range bookId {
d.books.Find(bson.M{"_id": id}).One(&books[i])
booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL)
booksColl.Find(bson.M{"_id": id}).One(&books[i])
books[i].Id = bson.ObjectId(books[i].Id).Hex()
}
return
@ -210,7 +214,8 @@ func (d *DB) GetNewBooks(r ...int) (books []Book, num int, err error) {
func (d *DB) BookActive(id bson.ObjectId) bool {
var book Book
err := d.books.Find(bson.M{"_id": id}).One(&book)
booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL)
err := booksColl.Find(bson.M{"_id": id}).One(&book)
if err != nil {
return false
}
@ -222,7 +227,9 @@ func (d *DB) GetFS(prefix string) *mgo.GridFS {
}
func (d *DB) GetTags(numTags int) ([]string, error) {
return d.mr.GetTags(numTags, d.books)
booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL)
mr := NewMR(d.session.DB(DB_NAME))
return mr.GetTags(numTags, booksColl)
}
type Visits struct {
@ -231,13 +238,19 @@ type Visits struct {
}
func (d *DB) GetHourVisits(start time.Time) ([]Visits, error) {
return d.mr.GetHourVisits(start, d.stats)
statsColl := d.session.DB(DB_NAME).C(STATS_COLL)
mr := NewMR(d.session.DB(DB_NAME))
return mr.GetHourVisits(start, statsColl)
}
func (d *DB) GetDayVisits(start time.Time) ([]Visits, error) {
return d.mr.GetDayVisits(start, d.stats)
statsColl := d.session.DB(DB_NAME).C(STATS_COLL)
mr := NewMR(d.session.DB(DB_NAME))
return mr.GetDayVisits(start, statsColl)
}
func (d *DB) GetMonthVisits(start time.Time) ([]Visits, error) {
return d.mr.GetMonthVisits(start, d.stats)
statsColl := d.session.DB(DB_NAME).C(STATS_COLL)
mr := NewMR(d.session.DB(DB_NAME))
return mr.GetMonthVisits(start, statsColl)
}

57
mapreduce.go
View file

@ -7,30 +7,12 @@ import (
)
type MR struct {
meta *mgo.Collection
tags *mgo.Collection
visited *mgo.Collection
downloaded *mgo.Collection
hourly_raw *mgo.Collection
daily_raw *mgo.Collection
monthly_raw *mgo.Collection
hourly *mgo.Collection
daily *mgo.Collection
monthly *mgo.Collection
database *mgo.Database
}
func NewMR(database *mgo.Database) *MR {
m := new(MR)
m.meta = database.C(META_COLL)
m.tags = database.C(TAGS_COLL)
m.visited = database.C(VISITED_COLL)
m.downloaded = database.C(DOWNLOADED_COLL)
m.hourly_raw = database.C(HOURLY_VISITS_COLL + "_raw")
m.daily_raw = database.C(DAILY_VISITS_COLL + "_raw")
m.monthly_raw = database.C(MONTHLY_VISITS_COLL + "_raw")
m.hourly = database.C(HOURLY_VISITS_COLL)
m.daily = database.C(DAILY_VISITS_COLL)
m.monthly = database.C(MONTHLY_VISITS_COLL)
m.database = database
return m
}
@ -56,7 +38,8 @@ func (m *MR) GetTags(numTags int, booksColl *mgo.Collection) ([]string, error) {
var result []struct {
Tag string "_id"
}
err := m.tags.Find(nil).Sort("-value").Limit(numTags).All(&result)
tagsColl := m.database.C(TAGS_COLL)
err := tagsColl.Find(nil).Sort("-value").Limit(numTags).All(&result)
if err != nil {
return nil, err
}
@ -88,7 +71,8 @@ func (m *MR) GetMostVisited(num int, statsColl *mgo.Collection) ([]bson.ObjectId
var result []struct {
Book bson.ObjectId "_id"
}
err := m.visited.Find(nil).Sort("-value").Limit(num).All(&result)
visitedColl := m.database.C(VISITED_COLL)
err := visitedColl.Find(nil).Sort("-value").Limit(num).All(&result)
if err != nil {
return nil, err
}
@ -120,7 +104,8 @@ func (m *MR) GetMostDownloaded(num int, statsColl *mgo.Collection) ([]bson.Objec
var result []struct {
Book bson.ObjectId "_id"
}
err := m.downloaded.Find(nil).Sort("-value").Limit(num).All(&result)
downloadedColl := m.database.C(DOWNLOADED_COLL)
err := downloadedColl.Find(nil).Sort("-value").Limit(num).All(&result)
if err != nil {
return nil, err
}
@ -157,14 +142,16 @@ func (m *MR) GetHourVisits(start time.Time, statsColl *mgo.Collection) ([]Visits
emit(this['_id']['date'], 1);
}`
mr2.Reduce = reduce
err = m.update(&mr2, bson.M{}, m.hourly_raw, HOURLY_VISITS_COLL)
hourly_raw := m.database.C(HOURLY_VISITS_COLL + "_raw")
err = m.update(&mr2, bson.M{}, hourly_raw, HOURLY_VISITS_COLL)
if err != nil {
return nil, err
}
}
var result []Visits
err := m.hourly.Find(nil).All(&result)
hourlyColl := m.database.C(HOURLY_VISITS_COLL)
err := hourlyColl.Find(nil).All(&result)
return result, err
}
@ -192,14 +179,16 @@ func (m *MR) GetDayVisits(start time.Time, statsColl *mgo.Collection) ([]Visits,
emit(this['_id']['date'], 1);
}`
mr2.Reduce = reduce
err = m.update(&mr2, bson.M{}, m.daily_raw, DAILY_VISITS_COLL)
daily_raw := m.database.C(DAILY_VISITS_COLL + "_raw")
err = m.update(&mr2, bson.M{}, daily_raw, DAILY_VISITS_COLL)
if err != nil {
return nil, err
}
}
var result []Visits
err := m.daily.Find(nil).All(&result)
dailyColl := m.database.C(DAILY_VISITS_COLL)
err := dailyColl.Find(nil).All(&result)
return result, err
}
@ -226,19 +215,22 @@ func (m *MR) GetMonthVisits(start time.Time, statsColl *mgo.Collection) ([]Visit
emit(this['_id']['date'], 1);
}`
mr2.Reduce = reduce
err = m.update(&mr2, bson.M{}, m.monthly_raw, MONTHLY_VISITS_COLL)
monthly_raw := m.database.C(MONTHLY_VISITS_COLL + "_raw")
err = m.update(&mr2, bson.M{}, monthly_raw, MONTHLY_VISITS_COLL)
if err != nil {
return nil, err
}
}
var result []Visits
err := m.monthly.Find(nil).All(&result)
monthlyColl := m.database.C(MONTHLY_VISITS_COLL)
err := monthlyColl.Find(nil).All(&result)
return result, err
}
func (m *MR) update(mr *mgo.MapReduce, query bson.M, queryColl *mgo.Collection, storeColl string) error {
_, err := m.meta.RemoveAll(bson.M{"type": storeColl})
metaColl := m.database.C(META_COLL)
_, err := metaColl.RemoveAll(bson.M{"type": storeColl})
if err != nil {
return err
}
@ -249,14 +241,15 @@ func (m *MR) update(mr *mgo.MapReduce, query bson.M, queryColl *mgo.Collection,
return err
}
return m.meta.Insert(bson.M{"type": storeColl})
return metaColl.Insert(bson.M{"type": storeColl})
}
func (m *MR) isOutdated(coll string, minutes float64) bool {
var result struct {
Id bson.ObjectId `bson:"_id"`
}
err := m.meta.Find(bson.M{"type": coll}).One(&result)
metaColl := m.database.C(META_COLL)
err := metaColl.Find(bson.M{"type": coll}).One(&result)
if err != nil {
return true
}