archives/trantor
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Clean up code

Browse source
This commit is contained in:
Las Zenow 2014-02-18 22:16:38 +01:00
parent e4726661b8
commit 8927a8cd6d
3 changed files with 89 additions and 114 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

79
database.go
View file

@ -176,14 +176,14 @@ func (d *DB) GetBooks(query bson.M, r ...int) (books []Book, num int, err error)
/* Get the most visited books /* Get the most visited books
*/ */
func (d *DB) GetVisitedBooks(num int) (books []Book, err error) { func (d *DB) GetVisitedBooks() (books []Book, err error) {
visitedColl := d.session.DB(DB_NAME).C(VISITED_COLL) visitedColl := d.session.DB(DB_NAME).C(VISITED_COLL)
bookId, err := GetBooksVisited(num, visitedColl) bookId, err := GetBooksVisited(visitedColl)
if err != nil { if err != nil {
return nil, err return nil, err
} }
books = make([]Book, num) books = make([]Book, len(bookId))
for i, id := range bookId { for i, id := range bookId {
booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL) booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL)
booksColl.Find(bson.M{"_id": id}).One(&books[i]) booksColl.Find(bson.M{"_id": id}).One(&books[i])
@ -193,21 +193,22 @@ func (d *DB) GetVisitedBooks(num int) (books []Book, err error) {
} }
func (d *DB) UpdateMostVisited() error { func (d *DB) UpdateMostVisited() error {
statsColl := d.session.DB(DB_NAME).C(STATS_COLL) var u DBUpdate
mr := NewMR(d.session.DB(DB_NAME)) u.src = d.session.DB(DB_NAME).C(STATS_COLL)
return mr.UpdateMostVisited(statsColl) u.dst = d.session.DB(DB_NAME).C(VISITED_COLL)
return u.UpdateMostBooks("book")
} }
/* Get the most downloaded books /* Get the most downloaded books
*/ */
func (d *DB) GetDownloadedBooks(num int) (books []Book, err error) { func (d *DB) GetDownloadedBooks() (books []Book, err error) {
downloadedColl := d.session.DB(DB_NAME).C(DOWNLOADED_COLL) downloadedColl := d.session.DB(DB_NAME).C(DOWNLOADED_COLL)
bookId, err := GetBooksVisited(num, downloadedColl) bookId, err := GetBooksVisited(downloadedColl)
if err != nil { if err != nil {
return nil, err return nil, err
} }
books = make([]Book, num) books = make([]Book, len(bookId))
for i, id := range bookId { for i, id := range bookId {
booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL) booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL)
booksColl.Find(bson.M{"_id": id}).One(&books[i]) booksColl.Find(bson.M{"_id": id}).One(&books[i])
@ -217,9 +218,10 @@ func (d *DB) GetDownloadedBooks(num int) (books []Book, err error) {
} }
func (d *DB) UpdateDownloadedBooks() error { func (d *DB) UpdateDownloadedBooks() error {
statsColl := d.session.DB(DB_NAME).C(STATS_COLL) var u DBUpdate
mr := NewMR(d.session.DB(DB_NAME)) u.src = d.session.DB(DB_NAME).C(STATS_COLL)
return mr.UpdateMostDownloaded(statsColl) u.dst = d.session.DB(DB_NAME).C(DOWNLOADED_COLL)
return u.UpdateMostBooks("download")
} }
/* optional parameters: length and start index /* optional parameters: length and start index
@ -244,15 +246,16 @@ func (d *DB) GetFS(prefix string) *mgo.GridFS {
return d.session.DB(DB_NAME).GridFS(prefix) return d.session.DB(DB_NAME).GridFS(prefix)
} }
func (d *DB) GetTags(numTags int) ([]string, error) { func (d *DB) GetTags() ([]string, error) {
tagsColl := d.session.DB(DB_NAME).C(TAGS_COLL) tagsColl := d.session.DB(DB_NAME).C(TAGS_COLL)
return GetTags(numTags, tagsColl) return GetTags(tagsColl)
} }
func (d *DB) UpdateTags() error { func (d *DB) UpdateTags() error {
booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL) var u DBUpdate
mr := NewMR(d.session.DB(DB_NAME)) u.src = d.session.DB(DB_NAME).C(BOOKS_COLL)
return mr.UpdateTags(booksColl) u.dst = d.session.DB(DB_NAME).C(TAGS_COLL)
return u.UpdateTags()
} }
type Visits struct { type Visits struct {
@ -266,9 +269,10 @@ func (d *DB) GetHourVisits() ([]Visits, error) {
} }
func (d *DB) UpdateHourVisits() error { func (d *DB) UpdateHourVisits() error {
statsColl := d.session.DB(DB_NAME).C(STATS_COLL) var u DBUpdate
mr := NewMR(d.session.DB(DB_NAME)) u.src = d.session.DB(DB_NAME).C(STATS_COLL)
return mr.UpdateHourVisits(statsColl) u.dst = d.session.DB(DB_NAME).C(HOURLY_VISITS_COLL)
return u.UpdateHourVisits()
} }
func (d *DB) GetDayVisits() ([]Visits, error) { func (d *DB) GetDayVisits() ([]Visits, error) {
@ -277,9 +281,10 @@ func (d *DB) GetDayVisits() ([]Visits, error) {
} }
func (d *DB) UpdateDayVisits() error { func (d *DB) UpdateDayVisits() error {
statsColl := d.session.DB(DB_NAME).C(STATS_COLL) var u DBUpdate
mr := NewMR(d.session.DB(DB_NAME)) u.src = d.session.DB(DB_NAME).C(STATS_COLL)
return mr.UpdateDayVisits(statsColl) u.dst = d.session.DB(DB_NAME).C(DAILY_VISITS_COLL)
return u.UpdateDayVisits()
} }
func (d *DB) GetMonthVisits() ([]Visits, error) { func (d *DB) GetMonthVisits() ([]Visits, error) {
@ -288,9 +293,10 @@ func (d *DB) GetMonthVisits() ([]Visits, error) {
} }
func (d *DB) UpdateMonthVisits() error { func (d *DB) UpdateMonthVisits() error {
statsColl := d.session.DB(DB_NAME).C(STATS_COLL) var u DBUpdate
mr := NewMR(d.session.DB(DB_NAME)) u.src = d.session.DB(DB_NAME).C(STATS_COLL)
return mr.UpdateMonthVisits(statsColl) u.dst = d.session.DB(DB_NAME).C(MONTHLY_VISITS_COLL)
return u.UpdateMonthVisits()
} }
func (d *DB) GetHourDownloads() ([]Visits, error) { func (d *DB) GetHourDownloads() ([]Visits, error) {
@ -299,9 +305,10 @@ func (d *DB) GetHourDownloads() ([]Visits, error) {
} }
func (d *DB) UpdateHourDownloads() error { func (d *DB) UpdateHourDownloads() error {
statsColl := d.session.DB(DB_NAME).C(STATS_COLL) var u DBUpdate
mr := NewMR(d.session.DB(DB_NAME)) u.src = d.session.DB(DB_NAME).C(STATS_COLL)
return mr.UpdateHourDownloads(statsColl) u.dst = d.session.DB(DB_NAME).C(HOURLY_DOWNLOADS_COLL)
return u.UpdateHourDownloads()
} }
func (d *DB) GetDayDownloads() ([]Visits, error) { func (d *DB) GetDayDownloads() ([]Visits, error) {
@ -310,9 +317,10 @@ func (d *DB) GetDayDownloads() ([]Visits, error) {
} }
func (d *DB) UpdateDayDownloads() error { func (d *DB) UpdateDayDownloads() error {
statsColl := d.session.DB(DB_NAME).C(STATS_COLL) var u DBUpdate
mr := NewMR(d.session.DB(DB_NAME)) u.src = d.session.DB(DB_NAME).C(STATS_COLL)
return mr.UpdateDayDownloads(statsColl) u.dst = d.session.DB(DB_NAME).C(DAILY_DOWNLOADS_COLL)
return u.UpdateDayDownloads()
} }
func (d *DB) GetMonthDownloads() ([]Visits, error) { func (d *DB) GetMonthDownloads() ([]Visits, error) {
@ -321,7 +329,8 @@ func (d *DB) GetMonthDownloads() ([]Visits, error) {
} }
func (d *DB) UpdateMonthDownloads() error { func (d *DB) UpdateMonthDownloads() error {
statsColl := d.session.DB(DB_NAME).C(STATS_COLL) var u DBUpdate
mr := NewMR(d.session.DB(DB_NAME)) u.src = d.session.DB(DB_NAME).C(STATS_COLL)
return mr.UpdateMonthDownloads(statsColl) u.dst = d.session.DB(DB_NAME).C(MONTHLY_DOWNLOADS_COLL)
return u.UpdateMonthDownloads()
} }

118
mapreduce.go → db_stats.go
View file

@ -6,11 +6,16 @@ import (
"time" "time"
) )
func GetTags(numTags int, tagsColl *mgo.Collection) ([]string, error) { type DBUpdate struct {
src *mgo.Collection
dst *mgo.Collection
}
func GetTags(tagsColl *mgo.Collection) ([]string, error) {
var result []struct { var result []struct {
Tag string "_id" Tag string "_id"
} }
err := tagsColl.Find(nil).Sort("-count").Limit(numTags).All(&result) err := tagsColl.Find(nil).Sort("-count").All(&result)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -22,11 +27,11 @@ func GetTags(numTags int, tagsColl *mgo.Collection) ([]string, error) {
return tags, nil return tags, nil
} }
func GetBooksVisited(num int, visitedColl *mgo.Collection) ([]bson.ObjectId, error) { func GetBooksVisited(visitedColl *mgo.Collection) ([]bson.ObjectId, error) {
var result []struct { var result []struct {
Book bson.ObjectId "_id" Book bson.ObjectId "_id"
} }
err := visitedColl.Find(nil).Sort("-count").Limit(num).All(&result) err := visitedColl.Find(nil).Sort("-count").All(&result)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -44,22 +49,12 @@ func GetVisits(visitsColl *mgo.Collection) ([]Visits, error) {
return result, err return result, err
} }
type MR struct { func (u *DBUpdate) UpdateTags() error {
database *mgo.Database
}
func NewMR(database *mgo.Database) *MR {
m := new(MR)
m.database = database
return m
}
func (m *MR) UpdateTags(booksColl *mgo.Collection) error {
var tags []struct { var tags []struct {
Tag string "_id" Tag string "_id"
Count int "count" Count int "count"
} }
err := booksColl.Pipe([]bson.M{ err := u.src.Pipe([]bson.M{
{"$project": bson.M{"subject": 1}}, {"$project": bson.M{"subject": 1}},
{"$unwind": "$subject"}, {"$unwind": "$subject"},
{"$group": bson.M{"_id": "$subject", "count": bson.M{"$sum": 1}}}, {"$group": bson.M{"_id": "$subject", "count": bson.M{"$sum": 1}}},
@ -70,10 +65,9 @@ func (m *MR) UpdateTags(booksColl *mgo.Collection) error {
return err return err
} }
tagsColl := m.database.C(TAGS_COLL) u.dst.DropCollection()
tagsColl.DropCollection()
for _, tag := range tags { for _, tag := range tags {
err = tagsColl.Insert(tag) err = u.dst.Insert(tag)
if err != nil { if err != nil {
return err return err
} }
@ -81,15 +75,7 @@ func (m *MR) UpdateTags(booksColl *mgo.Collection) error {
return nil return nil
} }
func (m *MR) UpdateMostVisited(statsColl *mgo.Collection) error { func (u *DBUpdate) UpdateMostBooks(section string) error {
return m.updateMostBooks(statsColl, "book", VISITED_COLL)
}
func (m *MR) UpdateMostDownloaded(statsColl *mgo.Collection) error {
return m.updateMostBooks(statsColl, "download", DOWNLOADED_COLL)
}
func (m *MR) updateMostBooks(statsColl *mgo.Collection, section string, resColl string) error {
const numDays = 30 const numDays = 30
start := time.Now().UTC().Add(-numDays * 24 * time.Hour) start := time.Now().UTC().Add(-numDays * 24 * time.Hour)
@ -97,7 +83,7 @@ func (m *MR) updateMostBooks(statsColl *mgo.Collection, section string, resColl
Book string "_id" Book string "_id"
Count int "count" Count int "count"
} }
err := statsColl.Pipe([]bson.M{ err := u.src.Pipe([]bson.M{
{"$match": bson.M{"date": bson.M{"$gt": start}, "section": section}}, {"$match": bson.M{"date": bson.M{"$gt": start}, "section": section}},
{"$project": bson.M{"id": 1}}, {"$project": bson.M{"id": 1}},
{"$group": bson.M{"_id": "$id", "count": bson.M{"$sum": 1}}}, {"$group": bson.M{"_id": "$id", "count": bson.M{"$sum": 1}}},
@ -108,10 +94,9 @@ func (m *MR) updateMostBooks(statsColl *mgo.Collection, section string, resColl
return err return err
} }
coll := m.database.C(resColl) u.dst.DropCollection()
coll.DropCollection()
for _, book := range books { for _, book := range books {
err = coll.Insert(book) err = u.dst.Insert(book)
if err != nil { if err != nil {
return err return err
} }
@ -119,84 +104,83 @@ func (m *MR) updateMostBooks(statsColl *mgo.Collection, section string, resColl
return nil return nil
} }
func (m *MR) UpdateHourVisits(statsColl *mgo.Collection) error { func (u *DBUpdate) UpdateHourVisits() error {
f := func(t time.Time) time.Time { f := func(t time.Time) time.Time {
const span = time.Hour const span = time.Hour
return t.Add(span).Truncate(span) return t.Add(span).Truncate(span)
} }
const numDays = 2 const numDays = 2
spanStore := numDays * 24 * time.Hour spanStore := numDays * 24 * time.Hour
return m.updateVisits(f, spanStore, HOURLY_VISITS_COLL, true) return u.updateVisits(f, spanStore, HOURLY_VISITS_COLL, true)
} }
func (m *MR) UpdateDayVisits(statsColl *mgo.Collection) error { func (u *DBUpdate) UpdateDayVisits() error {
f := func(t time.Time) time.Time { f := func(t time.Time) time.Time {
const span = 24 * time.Hour const span = 24 * time.Hour
return t.Add(span).Truncate(span) return t.Add(span).Truncate(span)
} }
const numDays = 30 const numDays = 30
spanStore := numDays * 24 * time.Hour spanStore := numDays * 24 * time.Hour
return m.updateVisits(f, spanStore, DAILY_VISITS_COLL, true) return u.updateVisits(f, spanStore, DAILY_VISITS_COLL, true)
} }
func (m *MR) UpdateMonthVisits(statsColl *mgo.Collection) error { func (u *DBUpdate) UpdateMonthVisits() error {
f := func(t time.Time) time.Time { f := func(t time.Time) time.Time {
const span = 24 * time.Hour const span = 24 * time.Hour
return t.AddDate(0, 1, 1-t.Day()).Truncate(span) return t.AddDate(0, 1, 1-t.Day()).Truncate(span)
} }
const numDays = 365 const numDays = 365
spanStore := numDays * 24 * time.Hour spanStore := numDays * 24 * time.Hour
return m.updateVisits(f, spanStore, MONTHLY_VISITS_COLL, true) return u.updateVisits(f, spanStore, MONTHLY_VISITS_COLL, true)
} }
func (m *MR) UpdateHourDownloads(statsColl *mgo.Collection) error { func (u *DBUpdate) UpdateHourDownloads() error {
f := func(t time.Time) time.Time { f := func(t time.Time) time.Time {
const span = time.Hour const span = time.Hour
return t.Add(span).Truncate(span) return t.Add(span).Truncate(span)
} }
const numDays = 2 const numDays = 2
spanStore := numDays * 24 * time.Hour spanStore := numDays * 24 * time.Hour
return m.updateVisits(f, spanStore, HOURLY_DOWNLOADS_COLL, false) return u.updateVisits(f, spanStore, HOURLY_DOWNLOADS_COLL, false)
} }
func (m *MR) UpdateDayDownloads(statsColl *mgo.Collection) error { func (u *DBUpdate) UpdateDayDownloads() error {
f := func(t time.Time) time.Time { f := func(t time.Time) time.Time {
const span = 24 * time.Hour const span = 24 * time.Hour
return t.Add(span).Truncate(span) return t.Add(span).Truncate(span)
} }
const numDays = 30 const numDays = 30
spanStore := numDays * 24 * time.Hour spanStore := numDays * 24 * time.Hour
return m.updateVisits(f, spanStore, DAILY_DOWNLOADS_COLL, false) return u.updateVisits(f, spanStore, DAILY_DOWNLOADS_COLL, false)
} }
func (m *MR) UpdateMonthDownloads(statsColl *mgo.Collection) error { func (u *DBUpdate) UpdateMonthDownloads() error {
f := func(t time.Time) time.Time { f := func(t time.Time) time.Time {
const span = 24 * time.Hour const span = 24 * time.Hour
return t.AddDate(0, 1, 1-t.Day()).Truncate(span) return t.AddDate(0, 1, 1-t.Day()).Truncate(span)
} }
const numDays = 365 const numDays = 365
spanStore := numDays * 24 * time.Hour spanStore := numDays * 24 * time.Hour
return m.updateVisits(f, spanStore, MONTHLY_DOWNLOADS_COLL, false) return u.updateVisits(f, spanStore, MONTHLY_DOWNLOADS_COLL, false)
} }
func (m *MR) updateVisits(incTime func(time.Time) time.Time, spanStore time.Duration, coll string, useSession bool) error { func (u *DBUpdate) updateVisits(incTime func(time.Time) time.Time, spanStore time.Duration, coll string, useSession bool) error {
storeColl := m.database.C(coll) start := u.calculateStart(spanStore)
start := m.calculateStart(spanStore, storeColl)
for start.Before(time.Now().UTC()) { for start.Before(time.Now().UTC()) {
stop := incTime(start) stop := incTime(start)
var count int var count int
var err error var err error
if useSession { if useSession {
count = m.countVisits(start, stop) count = u.countVisits(start, stop)
} else { } else {
count, err = m.countDownloads(start, stop) count, err = u.countDownloads(start, stop)
} }
if err != nil { if err != nil {
return err return err
} }
err = storeColl.Insert(bson.M{"date": start, "count": count}) err = u.dst.Insert(bson.M{"date": start, "count": count})
if err != nil { if err != nil {
return err return err
} }
@ -204,29 +188,28 @@ func (m *MR) updateVisits(incTime func(time.Time) time.Time, spanStore time.Dura
start = stop start = stop
} }
_, err := storeColl.RemoveAll(bson.M{"date": bson.M{"$lt": time.Now().UTC().Add(-spanStore)}}) _, err := u.dst.RemoveAll(bson.M{"date": bson.M{"$lt": time.Now().UTC().Add(-spanStore)}})
return err return err
} }
func (m *MR) calculateStart(spanStore time.Duration, storeColl *mgo.Collection) time.Time { func (u *DBUpdate) calculateStart(spanStore time.Duration) time.Time {
var date struct { var date struct {
Id bson.ObjectId `bson:"_id"` Id bson.ObjectId `bson:"_id"`
Date time.Time `bson:"date"` Date time.Time `bson:"date"`
} }
err := storeColl.Find(bson.M{}).Sort("-date").One(&date) err := u.dst.Find(bson.M{}).Sort("-date").One(&date)
if err == nil { if err == nil {
storeColl.RemoveId(date.Id) u.dst.RemoveId(date.Id)
return date.Date return date.Date
} }
return time.Now().UTC().Add(-spanStore).Truncate(time.Hour) return time.Now().UTC().Add(-spanStore).Truncate(time.Hour)
} }
func (m *MR) countVisits(start time.Time, stop time.Time) int { func (u *DBUpdate) countVisits(start time.Time, stop time.Time) int {
statsColl := m.database.C(STATS_COLL)
var result struct { var result struct {
Count int "count" Count int "count"
} }
err := statsColl.Pipe([]bson.M{ err := u.src.Pipe([]bson.M{
{"$match": bson.M{"date": bson.M{"$gte": start, "$lt": stop}}}, {"$match": bson.M{"date": bson.M{"$gte": start, "$lt": stop}}},
{"$group": bson.M{"_id": "$session"}}, {"$group": bson.M{"_id": "$session"}},
{"$group": bson.M{"_id": 1, "count": bson.M{"$sum": 1}}}, {"$group": bson.M{"_id": 1, "count": bson.M{"$sum": 1}}},
@ -238,24 +221,7 @@ func (m *MR) countVisits(start time.Time, stop time.Time) int {
return result.Count return result.Count
} }
func (m *MR) countDownloads(start time.Time, stop time.Time) (int, error) { func (u *DBUpdate) countDownloads(start time.Time, stop time.Time) (int, error) {
query := bson.M{"date": bson.M{"$gte": start, "$lt": stop}, "section": "download"} query := bson.M{"date": bson.M{"$gte": start, "$lt": stop}, "section": "download"}
statsColl := m.database.C(STATS_COLL) return u.src.Find(query).Count()
return statsColl.Find(query).Count()
}
func (m *MR) update(mr *mgo.MapReduce, query bson.M, queryColl *mgo.Collection, storeColl string) error {
metaColl := m.database.C(META_COLL)
_, err := metaColl.RemoveAll(bson.M{"type": storeColl})
if err != nil {
return err
}
mr.Out = bson.M{"replace": storeColl}
_, err = queryColl.Find(query).MapReduce(mr, nil)
if err != nil {
return err
}
return metaColl.Insert(bson.M{"type": storeColl})
} }

6
trantor.go
View file

@ -127,12 +127,12 @@ type indexData struct {
func indexHandler(h handler) { func indexHandler(h handler) {
var data indexData var data indexData
data.Tags, _ = h.db.GetTags(TAGS_DISPLAY) data.Tags, _ = h.db.GetTags()
data.S = GetStatus(h) data.S = GetStatus(h)
data.S.Home = true data.S.Home = true
data.Books, data.Count, _ = h.db.GetBooks(bson.M{"active": true}, BOOKS_FRONT_PAGE) data.Books, data.Count, _ = h.db.GetBooks(bson.M{"active": true}, BOOKS_FRONT_PAGE)
data.VisitedBooks, _ = h.db.GetVisitedBooks(BOOKS_FRONT_PAGE) data.VisitedBooks, _ = h.db.GetVisitedBooks()
data.DownloadedBooks, _ = h.db.GetDownloadedBooks(BOOKS_FRONT_PAGE) data.DownloadedBooks, _ = h.db.GetDownloadedBooks()
data.News = getNews(1, DAYS_NEWS_INDEXPAGE, h.db) data.News = getNews(1, DAYS_NEWS_INDEXPAGE, h.db)
loadTemplate(h.w, "index", data) loadTemplate(h.w, "index", data)
} }