diff --git a/database.go b/database.go index 29b4fbb..36c6834 100644 --- a/database.go +++ b/database.go @@ -176,14 +176,14 @@ func (d *DB) GetBooks(query bson.M, r ...int) (books []Book, num int, err error) /* Get the most visited books */ -func (d *DB) GetVisitedBooks(num int) (books []Book, err error) { +func (d *DB) GetVisitedBooks() (books []Book, err error) { visitedColl := d.session.DB(DB_NAME).C(VISITED_COLL) - bookId, err := GetBooksVisited(num, visitedColl) + bookId, err := GetBooksVisited(visitedColl) if err != nil { return nil, err } - books = make([]Book, num) + books = make([]Book, len(bookId)) for i, id := range bookId { booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL) booksColl.Find(bson.M{"_id": id}).One(&books[i]) @@ -193,21 +193,22 @@ func (d *DB) GetVisitedBooks(num int) (books []Book, err error) { } func (d *DB) UpdateMostVisited() error { - statsColl := d.session.DB(DB_NAME).C(STATS_COLL) - mr := NewMR(d.session.DB(DB_NAME)) - return mr.UpdateMostVisited(statsColl) + var u DBUpdate + u.src = d.session.DB(DB_NAME).C(STATS_COLL) + u.dst = d.session.DB(DB_NAME).C(VISITED_COLL) + return u.UpdateMostBooks("book") } /* Get the most downloaded books */ -func (d *DB) GetDownloadedBooks(num int) (books []Book, err error) { +func (d *DB) GetDownloadedBooks() (books []Book, err error) { downloadedColl := d.session.DB(DB_NAME).C(DOWNLOADED_COLL) - bookId, err := GetBooksVisited(num, downloadedColl) + bookId, err := GetBooksVisited(downloadedColl) if err != nil { return nil, err } - books = make([]Book, num) + books = make([]Book, len(bookId)) for i, id := range bookId { booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL) booksColl.Find(bson.M{"_id": id}).One(&books[i]) @@ -217,9 +218,10 @@ func (d *DB) GetDownloadedBooks(num int) (books []Book, err error) { } func (d *DB) UpdateDownloadedBooks() error { - statsColl := d.session.DB(DB_NAME).C(STATS_COLL) - mr := NewMR(d.session.DB(DB_NAME)) - return mr.UpdateMostDownloaded(statsColl) + var u DBUpdate + u.src = d.session.DB(DB_NAME).C(STATS_COLL) + u.dst = d.session.DB(DB_NAME).C(DOWNLOADED_COLL) + return u.UpdateMostBooks("download") } /* optional parameters: length and start index @@ -244,15 +246,16 @@ func (d *DB) GetFS(prefix string) *mgo.GridFS { return d.session.DB(DB_NAME).GridFS(prefix) } -func (d *DB) GetTags(numTags int) ([]string, error) { +func (d *DB) GetTags() ([]string, error) { tagsColl := d.session.DB(DB_NAME).C(TAGS_COLL) - return GetTags(numTags, tagsColl) + return GetTags(tagsColl) } func (d *DB) UpdateTags() error { - booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL) - mr := NewMR(d.session.DB(DB_NAME)) - return mr.UpdateTags(booksColl) + var u DBUpdate + u.src = d.session.DB(DB_NAME).C(BOOKS_COLL) + u.dst = d.session.DB(DB_NAME).C(TAGS_COLL) + return u.UpdateTags() } type Visits struct { @@ -266,9 +269,10 @@ func (d *DB) GetHourVisits() ([]Visits, error) { } func (d *DB) UpdateHourVisits() error { - statsColl := d.session.DB(DB_NAME).C(STATS_COLL) - mr := NewMR(d.session.DB(DB_NAME)) - return mr.UpdateHourVisits(statsColl) + var u DBUpdate + u.src = d.session.DB(DB_NAME).C(STATS_COLL) + u.dst = d.session.DB(DB_NAME).C(HOURLY_VISITS_COLL) + return u.UpdateHourVisits() } func (d *DB) GetDayVisits() ([]Visits, error) { @@ -277,9 +281,10 @@ func (d *DB) GetDayVisits() ([]Visits, error) { } func (d *DB) UpdateDayVisits() error { - statsColl := d.session.DB(DB_NAME).C(STATS_COLL) - mr := NewMR(d.session.DB(DB_NAME)) - return mr.UpdateDayVisits(statsColl) + var u DBUpdate + u.src = d.session.DB(DB_NAME).C(STATS_COLL) + u.dst = d.session.DB(DB_NAME).C(DAILY_VISITS_COLL) + return u.UpdateDayVisits() } func (d *DB) GetMonthVisits() ([]Visits, error) { @@ -288,9 +293,10 @@ func (d *DB) GetMonthVisits() ([]Visits, error) { } func (d *DB) UpdateMonthVisits() error { - statsColl := d.session.DB(DB_NAME).C(STATS_COLL) - mr := NewMR(d.session.DB(DB_NAME)) - return mr.UpdateMonthVisits(statsColl) + var u DBUpdate + u.src = d.session.DB(DB_NAME).C(STATS_COLL) + u.dst = d.session.DB(DB_NAME).C(MONTHLY_VISITS_COLL) + return u.UpdateMonthVisits() } func (d *DB) GetHourDownloads() ([]Visits, error) { @@ -299,9 +305,10 @@ func (d *DB) GetHourDownloads() ([]Visits, error) { } func (d *DB) UpdateHourDownloads() error { - statsColl := d.session.DB(DB_NAME).C(STATS_COLL) - mr := NewMR(d.session.DB(DB_NAME)) - return mr.UpdateHourDownloads(statsColl) + var u DBUpdate + u.src = d.session.DB(DB_NAME).C(STATS_COLL) + u.dst = d.session.DB(DB_NAME).C(HOURLY_DOWNLOADS_COLL) + return u.UpdateHourDownloads() } func (d *DB) GetDayDownloads() ([]Visits, error) { @@ -310,9 +317,10 @@ func (d *DB) GetDayDownloads() ([]Visits, error) { } func (d *DB) UpdateDayDownloads() error { - statsColl := d.session.DB(DB_NAME).C(STATS_COLL) - mr := NewMR(d.session.DB(DB_NAME)) - return mr.UpdateDayDownloads(statsColl) + var u DBUpdate + u.src = d.session.DB(DB_NAME).C(STATS_COLL) + u.dst = d.session.DB(DB_NAME).C(DAILY_DOWNLOADS_COLL) + return u.UpdateDayDownloads() } func (d *DB) GetMonthDownloads() ([]Visits, error) { @@ -321,7 +329,8 @@ func (d *DB) GetMonthDownloads() ([]Visits, error) { } func (d *DB) UpdateMonthDownloads() error { - statsColl := d.session.DB(DB_NAME).C(STATS_COLL) - mr := NewMR(d.session.DB(DB_NAME)) - return mr.UpdateMonthDownloads(statsColl) + var u DBUpdate + u.src = d.session.DB(DB_NAME).C(STATS_COLL) + u.dst = d.session.DB(DB_NAME).C(MONTHLY_DOWNLOADS_COLL) + return u.UpdateMonthDownloads() } diff --git a/mapreduce.go b/db_stats.go similarity index 51% rename from mapreduce.go rename to db_stats.go index 83ed2f9..1394e81 100644 --- a/mapreduce.go +++ b/db_stats.go @@ -6,11 +6,16 @@ import ( "time" ) -func GetTags(numTags int, tagsColl *mgo.Collection) ([]string, error) { +type DBUpdate struct { + src *mgo.Collection + dst *mgo.Collection +} + +func GetTags(tagsColl *mgo.Collection) ([]string, error) { var result []struct { Tag string "_id" } - err := tagsColl.Find(nil).Sort("-count").Limit(numTags).All(&result) + err := tagsColl.Find(nil).Sort("-count").All(&result) if err != nil { return nil, err } @@ -22,11 +27,11 @@ func GetTags(numTags int, tagsColl *mgo.Collection) ([]string, error) { return tags, nil } -func GetBooksVisited(num int, visitedColl *mgo.Collection) ([]bson.ObjectId, error) { +func GetBooksVisited(visitedColl *mgo.Collection) ([]bson.ObjectId, error) { var result []struct { Book bson.ObjectId "_id" } - err := visitedColl.Find(nil).Sort("-count").Limit(num).All(&result) + err := visitedColl.Find(nil).Sort("-count").All(&result) if err != nil { return nil, err } @@ -44,22 +49,12 @@ func GetVisits(visitsColl *mgo.Collection) ([]Visits, error) { return result, err } -type MR struct { - database *mgo.Database -} - -func NewMR(database *mgo.Database) *MR { - m := new(MR) - m.database = database - return m -} - -func (m *MR) UpdateTags(booksColl *mgo.Collection) error { +func (u *DBUpdate) UpdateTags() error { var tags []struct { Tag string "_id" Count int "count" } - err := booksColl.Pipe([]bson.M{ + err := u.src.Pipe([]bson.M{ {"$project": bson.M{"subject": 1}}, {"$unwind": "$subject"}, {"$group": bson.M{"_id": "$subject", "count": bson.M{"$sum": 1}}}, @@ -70,10 +65,9 @@ func (m *MR) UpdateTags(booksColl *mgo.Collection) error { return err } - tagsColl := m.database.C(TAGS_COLL) - tagsColl.DropCollection() + u.dst.DropCollection() for _, tag := range tags { - err = tagsColl.Insert(tag) + err = u.dst.Insert(tag) if err != nil { return err } @@ -81,15 +75,7 @@ func (m *MR) UpdateTags(booksColl *mgo.Collection) error { return nil } -func (m *MR) UpdateMostVisited(statsColl *mgo.Collection) error { - return m.updateMostBooks(statsColl, "book", VISITED_COLL) -} - -func (m *MR) UpdateMostDownloaded(statsColl *mgo.Collection) error { - return m.updateMostBooks(statsColl, "download", DOWNLOADED_COLL) -} - -func (m *MR) updateMostBooks(statsColl *mgo.Collection, section string, resColl string) error { +func (u *DBUpdate) UpdateMostBooks(section string) error { const numDays = 30 start := time.Now().UTC().Add(-numDays * 24 * time.Hour) @@ -97,7 +83,7 @@ func (m *MR) updateMostBooks(statsColl *mgo.Collection, section string, resColl Book string "_id" Count int "count" } - err := statsColl.Pipe([]bson.M{ + err := u.src.Pipe([]bson.M{ {"$match": bson.M{"date": bson.M{"$gt": start}, "section": section}}, {"$project": bson.M{"id": 1}}, {"$group": bson.M{"_id": "$id", "count": bson.M{"$sum": 1}}}, @@ -108,10 +94,9 @@ func (m *MR) updateMostBooks(statsColl *mgo.Collection, section string, resColl return err } - coll := m.database.C(resColl) - coll.DropCollection() + u.dst.DropCollection() for _, book := range books { - err = coll.Insert(book) + err = u.dst.Insert(book) if err != nil { return err } @@ -119,84 +104,83 @@ func (m *MR) updateMostBooks(statsColl *mgo.Collection, section string, resColl return nil } -func (m *MR) UpdateHourVisits(statsColl *mgo.Collection) error { +func (u *DBUpdate) UpdateHourVisits() error { f := func(t time.Time) time.Time { const span = time.Hour return t.Add(span).Truncate(span) } const numDays = 2 spanStore := numDays * 24 * time.Hour - return m.updateVisits(f, spanStore, HOURLY_VISITS_COLL, true) + return u.updateVisits(f, spanStore, HOURLY_VISITS_COLL, true) } -func (m *MR) UpdateDayVisits(statsColl *mgo.Collection) error { +func (u *DBUpdate) UpdateDayVisits() error { f := func(t time.Time) time.Time { const span = 24 * time.Hour return t.Add(span).Truncate(span) } const numDays = 30 spanStore := numDays * 24 * time.Hour - return m.updateVisits(f, spanStore, DAILY_VISITS_COLL, true) + return u.updateVisits(f, spanStore, DAILY_VISITS_COLL, true) } -func (m *MR) UpdateMonthVisits(statsColl *mgo.Collection) error { +func (u *DBUpdate) UpdateMonthVisits() error { f := func(t time.Time) time.Time { const span = 24 * time.Hour return t.AddDate(0, 1, 1-t.Day()).Truncate(span) } const numDays = 365 spanStore := numDays * 24 * time.Hour - return m.updateVisits(f, spanStore, MONTHLY_VISITS_COLL, true) + return u.updateVisits(f, spanStore, MONTHLY_VISITS_COLL, true) } -func (m *MR) UpdateHourDownloads(statsColl *mgo.Collection) error { +func (u *DBUpdate) UpdateHourDownloads() error { f := func(t time.Time) time.Time { const span = time.Hour return t.Add(span).Truncate(span) } const numDays = 2 spanStore := numDays * 24 * time.Hour - return m.updateVisits(f, spanStore, HOURLY_DOWNLOADS_COLL, false) + return u.updateVisits(f, spanStore, HOURLY_DOWNLOADS_COLL, false) } -func (m *MR) UpdateDayDownloads(statsColl *mgo.Collection) error { +func (u *DBUpdate) UpdateDayDownloads() error { f := func(t time.Time) time.Time { const span = 24 * time.Hour return t.Add(span).Truncate(span) } const numDays = 30 spanStore := numDays * 24 * time.Hour - return m.updateVisits(f, spanStore, DAILY_DOWNLOADS_COLL, false) + return u.updateVisits(f, spanStore, DAILY_DOWNLOADS_COLL, false) } -func (m *MR) UpdateMonthDownloads(statsColl *mgo.Collection) error { +func (u *DBUpdate) UpdateMonthDownloads() error { f := func(t time.Time) time.Time { const span = 24 * time.Hour return t.AddDate(0, 1, 1-t.Day()).Truncate(span) } const numDays = 365 spanStore := numDays * 24 * time.Hour - return m.updateVisits(f, spanStore, MONTHLY_DOWNLOADS_COLL, false) + return u.updateVisits(f, spanStore, MONTHLY_DOWNLOADS_COLL, false) } -func (m *MR) updateVisits(incTime func(time.Time) time.Time, spanStore time.Duration, coll string, useSession bool) error { - storeColl := m.database.C(coll) - start := m.calculateStart(spanStore, storeColl) +func (u *DBUpdate) updateVisits(incTime func(time.Time) time.Time, spanStore time.Duration, coll string, useSession bool) error { + start := u.calculateStart(spanStore) for start.Before(time.Now().UTC()) { stop := incTime(start) var count int var err error if useSession { - count = m.countVisits(start, stop) + count = u.countVisits(start, stop) } else { - count, err = m.countDownloads(start, stop) + count, err = u.countDownloads(start, stop) } if err != nil { return err } - err = storeColl.Insert(bson.M{"date": start, "count": count}) + err = u.dst.Insert(bson.M{"date": start, "count": count}) if err != nil { return err } @@ -204,29 +188,28 @@ func (m *MR) updateVisits(incTime func(time.Time) time.Time, spanStore time.Dura start = stop } - _, err := storeColl.RemoveAll(bson.M{"date": bson.M{"$lt": time.Now().UTC().Add(-spanStore)}}) + _, err := u.dst.RemoveAll(bson.M{"date": bson.M{"$lt": time.Now().UTC().Add(-spanStore)}}) return err } -func (m *MR) calculateStart(spanStore time.Duration, storeColl *mgo.Collection) time.Time { +func (u *DBUpdate) calculateStart(spanStore time.Duration) time.Time { var date struct { Id bson.ObjectId `bson:"_id"` Date time.Time `bson:"date"` } - err := storeColl.Find(bson.M{}).Sort("-date").One(&date) + err := u.dst.Find(bson.M{}).Sort("-date").One(&date) if err == nil { - storeColl.RemoveId(date.Id) + u.dst.RemoveId(date.Id) return date.Date } return time.Now().UTC().Add(-spanStore).Truncate(time.Hour) } -func (m *MR) countVisits(start time.Time, stop time.Time) int { - statsColl := m.database.C(STATS_COLL) +func (u *DBUpdate) countVisits(start time.Time, stop time.Time) int { var result struct { Count int "count" } - err := statsColl.Pipe([]bson.M{ + err := u.src.Pipe([]bson.M{ {"$match": bson.M{"date": bson.M{"$gte": start, "$lt": stop}}}, {"$group": bson.M{"_id": "$session"}}, {"$group": bson.M{"_id": 1, "count": bson.M{"$sum": 1}}}, @@ -238,24 +221,7 @@ func (m *MR) countVisits(start time.Time, stop time.Time) int { return result.Count } -func (m *MR) countDownloads(start time.Time, stop time.Time) (int, error) { +func (u *DBUpdate) countDownloads(start time.Time, stop time.Time) (int, error) { query := bson.M{"date": bson.M{"$gte": start, "$lt": stop}, "section": "download"} - statsColl := m.database.C(STATS_COLL) - return statsColl.Find(query).Count() -} - -func (m *MR) update(mr *mgo.MapReduce, query bson.M, queryColl *mgo.Collection, storeColl string) error { - metaColl := m.database.C(META_COLL) - _, err := metaColl.RemoveAll(bson.M{"type": storeColl}) - if err != nil { - return err - } - - mr.Out = bson.M{"replace": storeColl} - _, err = queryColl.Find(query).MapReduce(mr, nil) - if err != nil { - return err - } - - return metaColl.Insert(bson.M{"type": storeColl}) + return u.src.Find(query).Count() } diff --git a/trantor.go b/trantor.go index 36560ce..a226b74 100644 --- a/trantor.go +++ b/trantor.go @@ -127,12 +127,12 @@ type indexData struct { func indexHandler(h handler) { var data indexData - data.Tags, _ = h.db.GetTags(TAGS_DISPLAY) + data.Tags, _ = h.db.GetTags() data.S = GetStatus(h) data.S.Home = true data.Books, data.Count, _ = h.db.GetBooks(bson.M{"active": true}, BOOKS_FRONT_PAGE) - data.VisitedBooks, _ = h.db.GetVisitedBooks(BOOKS_FRONT_PAGE) - data.DownloadedBooks, _ = h.db.GetDownloadedBooks(BOOKS_FRONT_PAGE) + data.VisitedBooks, _ = h.db.GetVisitedBooks() + data.DownloadedBooks, _ = h.db.GetDownloadedBooks() data.News = getNews(1, DAYS_NEWS_INDEXPAGE, h.db) loadTemplate(h.w, "index", data) }