From 6ae0d1b0ac85bdc826d16fc7847b97613b54bfe0 Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Fri, 14 Feb 2014 19:32:08 +0100 Subject: [PATCH 01/11] Fix field name on import --- tools/importer/importer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/importer/importer.go b/tools/importer/importer.go index b31282f..293bd0c 100644 --- a/tools/importer/importer.go +++ b/tools/importer/importer.go @@ -40,8 +40,8 @@ func uploadEpub(filename string, db *DB) { return } - book["filename"] = id - book["filenamesize"] = size + book["file"] = id + book["filesize"] = size err = db.InsertBook(book) if err != nil { log.Error("Error storing metadata (", title, "): ", err) From a6421d0dd34b7878b6f4ee56e0c362b701695ebf Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Mon, 17 Feb 2014 21:24:42 +0100 Subject: [PATCH 02/11] Most visited/downloaded books are calculated only for the last 30 days --- mapreduce.go | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/mapreduce.go b/mapreduce.go index c24deec..012dd3b 100644 --- a/mapreduce.go +++ b/mapreduce.go @@ -70,21 +70,17 @@ func (m *MR) UpdateTags(booksColl *mgo.Collection) error { } func (m *MR) UpdateMostVisited(statsColl *mgo.Collection) error { - var mr mgo.MapReduce - mr.Map = `function() { - if (this.id) { - emit(this.id, 1); - } - }` - mr.Reduce = `function(tag, vals) { - var count = 0; - vals.forEach(function() { count += 1; }); - return count; - }` - return m.update(&mr, bson.M{"section": "book"}, statsColl, VISITED_COLL) + return m.updateMostBooks(statsColl, "book", VISITED_COLL) } func (m *MR) UpdateMostDownloaded(statsColl *mgo.Collection) error { + return m.updateMostBooks(statsColl, "download", DOWNLOADED_COLL) +} + +func (m *MR) updateMostBooks(statsColl *mgo.Collection, section string, resColl string) error { + const numDays = 30 + start := time.Now().UTC().Add(-numDays * 24 * time.Hour) + var mr mgo.MapReduce mr.Map = `function() { emit(this.id, 1); @@ -94,7 +90,7 @@ func (m *MR) UpdateMostDownloaded(statsColl *mgo.Collection) error { vals.forEach(function() { count += 1; }); return count; }` - return m.update(&mr, bson.M{"section": "download"}, statsColl, DOWNLOADED_COLL) + return m.update(&mr, bson.M{"date": bson.M{"$gt": start}, "section": section}, statsColl, resColl) } func (m *MR) UpdateHourVisits(statsColl *mgo.Collection) error { @@ -233,7 +229,6 @@ func (m *MR) UpdateDayDownloads(statsColl *mgo.Collection) error { func (m *MR) UpdateMonthDownloads(statsColl *mgo.Collection) error { const numDays = 365 - start := time.Now().UTC().Add(-numDays * 24 * time.Hour).Truncate(24 * time.Hour) var mr mgo.MapReduce From 4f4c82f7e0c7d88827e864f6fe5aba1ef04bed47 Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Tue, 18 Feb 2014 00:20:27 +0100 Subject: [PATCH 03/11] For download statistics select the section on the query --- mapreduce.go | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/mapreduce.go b/mapreduce.go index 012dd3b..9e6aa26 100644 --- a/mapreduce.go +++ b/mapreduce.go @@ -190,20 +190,18 @@ func (m *MR) UpdateHourDownloads(statsColl *mgo.Collection) error { var mr mgo.MapReduce mr.Map = `function() { - if (this.section == "download") { - var date = Date.UTC(this.date.getUTCFullYear(), - this.date.getUTCMonth(), - this.date.getUTCDate(), - this.date.getUTCHours()); - emit(date, 1); - } - }` + var date = Date.UTC(this.date.getUTCFullYear(), + this.date.getUTCMonth(), + this.date.getUTCDate(), + this.date.getUTCHours()); + emit(date, 1); + }` mr.Reduce = `function(date, vals) { var count = 0; vals.forEach(function(v) { count += v; }); return count; }` - return m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, HOURLY_DOWNLOADS_COLL) + return m.update(&mr, bson.M{"date": bson.M{"$gte": start}, "section": "download"}, statsColl, HOURLY_DOWNLOADS_COLL) } func (m *MR) UpdateDayDownloads(statsColl *mgo.Collection) error { @@ -212,19 +210,17 @@ func (m *MR) UpdateDayDownloads(statsColl *mgo.Collection) error { var mr mgo.MapReduce mr.Map = `function() { - if (this.section == "download") { - var date = Date.UTC(this.date.getUTCFullYear(), - this.date.getUTCMonth(), - this.date.getUTCDate()); - emit(date, 1); - } + var date = Date.UTC(this.date.getUTCFullYear(), + this.date.getUTCMonth(), + this.date.getUTCDate()); + emit(date, 1); }` mr.Reduce = `function(date, vals) { var count = 0; vals.forEach(function(v) { count += v; }); return count; }` - return m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, DAILY_DOWNLOADS_COLL) + return m.update(&mr, bson.M{"date": bson.M{"$gte": start}, "section": "download"}, statsColl, DAILY_DOWNLOADS_COLL) } func (m *MR) UpdateMonthDownloads(statsColl *mgo.Collection) error { @@ -233,18 +229,16 @@ func (m *MR) UpdateMonthDownloads(statsColl *mgo.Collection) error { var mr mgo.MapReduce mr.Map = `function() { - if (this.section == "download") { - var date = Date.UTC(this.date.getUTCFullYear(), - this.date.getUTCMonth()); - emit(date, 1); - } + var date = Date.UTC(this.date.getUTCFullYear(), + this.date.getUTCMonth()); + emit(date, 1); }` mr.Reduce = `function(date, vals) { var count = 0; vals.forEach(function(v) { count += v; }); return count; }` - return m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, MONTHLY_DOWNLOADS_COLL) + return m.update(&mr, bson.M{"date": bson.M{"$gte": start}, "section": "download"}, statsColl, MONTHLY_DOWNLOADS_COLL) } func (m *MR) update(mr *mgo.MapReduce, query bson.M, queryColl *mgo.Collection, storeColl string) error { From f2c728191cfd6a7d666d62d926580cf2c59798a3 Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Tue, 18 Feb 2014 00:47:58 +0100 Subject: [PATCH 04/11] Remove unused function isOutdated --- mapreduce.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/mapreduce.go b/mapreduce.go index 9e6aa26..cf69442 100644 --- a/mapreduce.go +++ b/mapreduce.go @@ -256,17 +256,3 @@ func (m *MR) update(mr *mgo.MapReduce, query bson.M, queryColl *mgo.Collection, return metaColl.Insert(bson.M{"type": storeColl}) } - -func (m *MR) isOutdated(coll string, minutes float64) bool { - var result struct { - Id bson.ObjectId `bson:"_id"` - } - metaColl := m.database.C(META_COLL) - err := metaColl.Find(bson.M{"type": coll}).One(&result) - if err != nil { - return true - } - - lastUpdate := result.Id.Time() - return time.Since(lastUpdate).Minutes() > minutes -} From 42de6bbf644e4890f8a99a7925aa9f77d52ea747 Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Tue, 18 Feb 2014 19:05:27 +0100 Subject: [PATCH 05/11] Convert stats mapreduce to aggregations --- database.go | 4 +- mapreduce.go | 223 +++++++++++++++++++++++---------------------------- stats.go | 12 +-- 3 files changed, 108 insertions(+), 131 deletions(-) diff --git a/database.go b/database.go index e49b644..29b4fbb 100644 --- a/database.go +++ b/database.go @@ -256,8 +256,8 @@ func (d *DB) UpdateTags() error { } type Visits struct { - Date int64 "_id" - Count int "value" + Date time.Time "date" + Count int "count" } func (d *DB) GetHourVisits() ([]Visits, error) { diff --git a/mapreduce.go b/mapreduce.go index cf69442..b8eb10f 100644 --- a/mapreduce.go +++ b/mapreduce.go @@ -94,151 +94,128 @@ func (m *MR) updateMostBooks(statsColl *mgo.Collection, section string, resColl } func (m *MR) UpdateHourVisits(statsColl *mgo.Collection) error { - const numDays = 2 - start := time.Now().UTC().Add(-numDays * 24 * time.Hour) - - const reduce = `function(date, vals) { - var count = 0; - vals.forEach(function(v) { count += v; }); - return count; - }` - var mr mgo.MapReduce - mr.Map = `function() { - var date = Date.UTC(this.date.getUTCFullYear(), - this.date.getUTCMonth(), - this.date.getUTCDate(), - this.date.getUTCHours()); - emit({date: date, session: this.session}, 1); - }` - mr.Reduce = reduce - err := m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, HOURLY_VISITS_COLL+"_raw") - if err != nil { - return err + f := func(t time.Time) time.Time { + const span = time.Hour + return t.Add(span).Truncate(span) } - var mr2 mgo.MapReduce - mr2.Map = `function() { - emit(this['_id']['date'], 1); - }` - mr2.Reduce = reduce - hourly_raw := m.database.C(HOURLY_VISITS_COLL + "_raw") - return m.update(&mr2, bson.M{}, hourly_raw, HOURLY_VISITS_COLL) + const numDays = 2 + spanStore := numDays * 24 * time.Hour + return m.updateVisits(f, spanStore, HOURLY_VISITS_COLL, true) } func (m *MR) UpdateDayVisits(statsColl *mgo.Collection) error { - const numDays = 30 - start := time.Now().UTC().Add(-numDays * 24 * time.Hour).Truncate(24 * time.Hour) - - const reduce = `function(date, vals) { - var count = 0; - vals.forEach(function(v) { count += v; }); - return count; - }` - var mr mgo.MapReduce - mr.Map = `function() { - var date = Date.UTC(this.date.getUTCFullYear(), - this.date.getUTCMonth(), - this.date.getUTCDate()); - emit({date: date, session: this.session}, 1); - }` - mr.Reduce = reduce - err := m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, DAILY_VISITS_COLL+"_raw") - if err != nil { - return err + f := func(t time.Time) time.Time { + const span = 24 * time.Hour + return t.Add(span).Truncate(span) } - var mr2 mgo.MapReduce - mr2.Map = `function() { - emit(this['_id']['date'], 1); - }` - mr2.Reduce = reduce - daily_raw := m.database.C(DAILY_VISITS_COLL + "_raw") - return m.update(&mr2, bson.M{}, daily_raw, DAILY_VISITS_COLL) + const numDays = 30 + spanStore := numDays * 24 * time.Hour + return m.updateVisits(f, spanStore, DAILY_VISITS_COLL, true) } func (m *MR) UpdateMonthVisits(statsColl *mgo.Collection) error { - const numDays = 365 - - start := time.Now().UTC().Add(-numDays * 24 * time.Hour).Truncate(24 * time.Hour) - - const reduce = `function(date, vals) { - var count = 0; - vals.forEach(function(v) { count += v; }); - return count; - }` - var mr mgo.MapReduce - mr.Map = `function() { - var date = Date.UTC(this.date.getUTCFullYear(), - this.date.getUTCMonth()); - emit({date: date, session: this.session}, 1); - }` - mr.Reduce = reduce - err := m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, MONTHLY_VISITS_COLL+"_raw") - if err != nil { - return err + f := func(t time.Time) time.Time { + const span = 24 * time.Hour + return t.AddDate(0, 1, 1-t.Day()).Truncate(span) } - var mr2 mgo.MapReduce - mr2.Map = `function() { - emit(this['_id']['date'], 1); - }` - mr2.Reduce = reduce - monthly_raw := m.database.C(MONTHLY_VISITS_COLL + "_raw") - return m.update(&mr2, bson.M{}, monthly_raw, MONTHLY_VISITS_COLL) + const numDays = 365 + spanStore := numDays * 24 * time.Hour + return m.updateVisits(f, spanStore, MONTHLY_VISITS_COLL, true) } func (m *MR) UpdateHourDownloads(statsColl *mgo.Collection) error { + f := func(t time.Time) time.Time { + const span = time.Hour + return t.Add(span).Truncate(span) + } const numDays = 2 - start := time.Now().UTC().Add(-numDays * 24 * time.Hour) - - var mr mgo.MapReduce - mr.Map = `function() { - var date = Date.UTC(this.date.getUTCFullYear(), - this.date.getUTCMonth(), - this.date.getUTCDate(), - this.date.getUTCHours()); - emit(date, 1); - }` - mr.Reduce = `function(date, vals) { - var count = 0; - vals.forEach(function(v) { count += v; }); - return count; - }` - return m.update(&mr, bson.M{"date": bson.M{"$gte": start}, "section": "download"}, statsColl, HOURLY_DOWNLOADS_COLL) + spanStore := numDays * 24 * time.Hour + return m.updateVisits(f, spanStore, HOURLY_DOWNLOADS_COLL, false) } func (m *MR) UpdateDayDownloads(statsColl *mgo.Collection) error { + f := func(t time.Time) time.Time { + const span = 24 * time.Hour + return t.Add(span).Truncate(span) + } const numDays = 30 - start := time.Now().UTC().Add(-numDays * 24 * time.Hour).Truncate(24 * time.Hour) - - var mr mgo.MapReduce - mr.Map = `function() { - var date = Date.UTC(this.date.getUTCFullYear(), - this.date.getUTCMonth(), - this.date.getUTCDate()); - emit(date, 1); - }` - mr.Reduce = `function(date, vals) { - var count = 0; - vals.forEach(function(v) { count += v; }); - return count; - }` - return m.update(&mr, bson.M{"date": bson.M{"$gte": start}, "section": "download"}, statsColl, DAILY_DOWNLOADS_COLL) + spanStore := numDays * 24 * time.Hour + return m.updateVisits(f, spanStore, DAILY_DOWNLOADS_COLL, false) } func (m *MR) UpdateMonthDownloads(statsColl *mgo.Collection) error { + f := func(t time.Time) time.Time { + const span = 24 * time.Hour + return t.AddDate(0, 1, 1-t.Day()).Truncate(span) + } const numDays = 365 - start := time.Now().UTC().Add(-numDays * 24 * time.Hour).Truncate(24 * time.Hour) + spanStore := numDays * 24 * time.Hour + return m.updateVisits(f, spanStore, MONTHLY_DOWNLOADS_COLL, false) +} - var mr mgo.MapReduce - mr.Map = `function() { - var date = Date.UTC(this.date.getUTCFullYear(), - this.date.getUTCMonth()); - emit(date, 1); - }` - mr.Reduce = `function(date, vals) { - var count = 0; - vals.forEach(function(v) { count += v; }); - return count; - }` - return m.update(&mr, bson.M{"date": bson.M{"$gte": start}, "section": "download"}, statsColl, MONTHLY_DOWNLOADS_COLL) +func (m *MR) updateVisits(incTime func(time.Time) time.Time, spanStore time.Duration, coll string, useSession bool) error { + storeColl := m.database.C(coll) + start := m.calculateStart(spanStore, storeColl) + for start.Before(time.Now().UTC()) { + stop := incTime(start) + + var count int + var err error + if useSession { + count = m.countVisits(start, stop) + } else { + count, err = m.countDownloads(start, stop) + } + if err != nil { + return err + } + + err = storeColl.Insert(bson.M{"date": start, "count": count}) + if err != nil { + return err + } + + start = stop + } + + _, err := storeColl.RemoveAll(bson.M{"date": bson.M{"$lt": time.Now().UTC().Add(-spanStore)}}) + return err +} + +func (m *MR) calculateStart(spanStore time.Duration, storeColl *mgo.Collection) time.Time { + var date struct { + Id bson.ObjectId `bson:"_id"` + Date time.Time `bson:"date"` + } + err := storeColl.Find(bson.M{}).Sort("-date").One(&date) + if err == nil { + storeColl.RemoveId(date.Id) + return date.Date + } + return time.Now().UTC().Add(-spanStore).Truncate(time.Hour) +} + +func (m *MR) countVisits(start time.Time, stop time.Time) int { + statsColl := m.database.C(STATS_COLL) + var result struct { + Count int "count" + } + err := statsColl.Pipe([]bson.M{ + {"$match": bson.M{"date": bson.M{"$gte": start, "$lt": stop}}}, + {"$group": bson.M{"_id": "$session"}}, + {"$group": bson.M{"_id": 1, "count": bson.M{"$sum": 1}}}, + }).One(&result) + if err != nil { + return 0 + } + + return result.Count +} + +func (m *MR) countDownloads(start time.Time, stop time.Time) (int, error) { + query := bson.M{"date": bson.M{"$gte": start, "$lt": stop}, "section": "download"} + statsColl := m.database.C(STATS_COLL) + return statsColl.Find(query).Count() } func (m *MR) update(mr *mgo.MapReduce, query bson.M, queryColl *mgo.Collection, storeColl string) error { diff --git a/stats.go b/stats.go index 4218ddf..626aafa 100644 --- a/stats.go +++ b/stats.go @@ -100,7 +100,7 @@ func getHourlyVisits(db *DB) []visitData { visit, _ := db.GetHourVisits() for _, v := range visit { var elem visitData - hour := time.Unix(v.Date/1000, 0).UTC().Hour() + hour := v.Date.UTC().Hour() elem.Label = strconv.Itoa(hour + 1) elem.Count = v.Count visits = append(visits, elem) @@ -115,7 +115,7 @@ func getDailyVisits(db *DB) []visitData { visit, _ := db.GetDayVisits() for _, v := range visit { var elem visitData - day := time.Unix(v.Date/1000, 0).UTC().Day() + day := v.Date.UTC().Day() elem.Label = strconv.Itoa(day) elem.Count = v.Count visits = append(visits, elem) @@ -130,7 +130,7 @@ func getMonthlyVisits(db *DB) []visitData { visit, _ := db.GetMonthVisits() for _, v := range visit { var elem visitData - month := time.Unix(v.Date/1000, 0).UTC().Month() + month := v.Date.UTC().Month() elem.Label = month.String() elem.Count = v.Count visits = append(visits, elem) @@ -145,7 +145,7 @@ func getHourlyDownloads(db *DB) []visitData { visit, _ := db.GetHourDownloads() for _, v := range visit { var elem visitData - hour := time.Unix(v.Date/1000, 0).UTC().Hour() + hour := v.Date.UTC().Hour() elem.Label = strconv.Itoa(hour + 1) elem.Count = v.Count visits = append(visits, elem) @@ -160,7 +160,7 @@ func getDailyDownloads(db *DB) []visitData { visit, _ := db.GetDayDownloads() for _, v := range visit { var elem visitData - day := time.Unix(v.Date/1000, 0).UTC().Day() + day := v.Date.UTC().Day() elem.Label = strconv.Itoa(day) elem.Count = v.Count visits = append(visits, elem) @@ -175,7 +175,7 @@ func getMonthlyDownloads(db *DB) []visitData { visit, _ := db.GetMonthDownloads() for _, v := range visit { var elem visitData - month := time.Unix(v.Date/1000, 0).UTC().Month() + month := v.Date.UTC().Month() elem.Label = month.String() elem.Count = v.Count visits = append(visits, elem) From b3fd8aabe25dbe4e839b1122652857f67f7ec367 Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Tue, 18 Feb 2014 19:05:48 +0100 Subject: [PATCH 06/11] Update dependencies --- README | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README b/README index ea5dbae..395f6e7 100644 --- a/README +++ b/README @@ -13,11 +13,9 @@ https://gitorious.org/trantor/ In order to run Trantor, you need to install the following packages: * Go language -* Epub development library * Mongodb * Bazaar -* Mercurial -* Git (necessary only if you want to deal with the repository) +* Git Under Debian Wheezy you can simply run: From 706fc88a61bb97911c758dce26607aed18303d2c Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Tue, 18 Feb 2014 21:35:26 +0100 Subject: [PATCH 07/11] Use aggregations to calculate tags --- mapreduce.go | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/mapreduce.go b/mapreduce.go index b8eb10f..9adc18f 100644 --- a/mapreduce.go +++ b/mapreduce.go @@ -10,7 +10,7 @@ func GetTags(numTags int, tagsColl *mgo.Collection) ([]string, error) { var result []struct { Tag string "_id" } - err := tagsColl.Find(nil).Sort("-value").Limit(numTags).All(&result) + err := tagsColl.Find(nil).Sort("-count").Limit(numTags).All(&result) if err != nil { return nil, err } @@ -55,18 +55,30 @@ func NewMR(database *mgo.Database) *MR { } func (m *MR) UpdateTags(booksColl *mgo.Collection) error { - var mr mgo.MapReduce - mr.Map = `function() { - if (this.subject) { - this.subject.forEach(function(s) { emit(s, 1); }); - } - }` - mr.Reduce = `function(tag, vals) { - var count = 0; - vals.forEach(function() { count += 1; }); - return count; - }` - return m.update(&mr, bson.M{"active": true}, booksColl, TAGS_COLL) + var tags []struct { + Tag string "_id" + Count int "count" + } + err := booksColl.Pipe([]bson.M{ + {"$project": bson.M{"subject": 1}}, + {"$unwind": "$subject"}, + {"$group": bson.M{"_id": "$subject", "count": bson.M{"$sum": 1}}}, + {"$sort": bson.M{"count": -1}}, + {"$limit": TAGS_DISPLAY}, + }).All(&tags) + if err != nil { + return err + } + + tagsColl := m.database.C(TAGS_COLL) + tagsColl.DropCollection() + for _, tag := range tags { + err = tagsColl.Insert(tag) + if err != nil { + return err + } + } + return nil } func (m *MR) UpdateMostVisited(statsColl *mgo.Collection) error { From e4726661b878fc9ebf8dcd0979bada7d462d6237 Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Tue, 18 Feb 2014 21:50:44 +0100 Subject: [PATCH 08/11] Use aggregation for most visited books --- config.go | 1 + mapreduce.go | 36 +++++++++++++++++++++++++----------- trantor.go | 6 +++--- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/config.go b/config.go index 6099708..e172e9c 100644 --- a/config.go +++ b/config.go @@ -34,6 +34,7 @@ const ( MINUTES_UPDATE_MONTHLY_D = 60*24 + 17 MINUTES_UPDATE_LOGGER = 5 TAGS_DISPLAY = 50 + BOOKS_FRONT_PAGE = 6 SEARCH_ITEMS_PAGE = 20 NEW_ITEMS_PAGE = 50 NUM_NEWS = 10 diff --git a/mapreduce.go b/mapreduce.go index 9adc18f..83ed2f9 100644 --- a/mapreduce.go +++ b/mapreduce.go @@ -26,7 +26,7 @@ func GetBooksVisited(num int, visitedColl *mgo.Collection) ([]bson.ObjectId, err var result []struct { Book bson.ObjectId "_id" } - err := visitedColl.Find(nil).Sort("-value").Limit(num).All(&result) + err := visitedColl.Find(nil).Sort("-count").Limit(num).All(&result) if err != nil { return nil, err } @@ -93,16 +93,30 @@ func (m *MR) updateMostBooks(statsColl *mgo.Collection, section string, resColl const numDays = 30 start := time.Now().UTC().Add(-numDays * 24 * time.Hour) - var mr mgo.MapReduce - mr.Map = `function() { - emit(this.id, 1); - }` - mr.Reduce = `function(tag, vals) { - var count = 0; - vals.forEach(function() { count += 1; }); - return count; - }` - return m.update(&mr, bson.M{"date": bson.M{"$gt": start}, "section": section}, statsColl, resColl) + var books []struct { + Book string "_id" + Count int "count" + } + err := statsColl.Pipe([]bson.M{ + {"$match": bson.M{"date": bson.M{"$gt": start}, "section": section}}, + {"$project": bson.M{"id": 1}}, + {"$group": bson.M{"_id": "$id", "count": bson.M{"$sum": 1}}}, + {"$sort": bson.M{"count": -1}}, + {"$limit": BOOKS_FRONT_PAGE}, + }).All(&books) + if err != nil { + return err + } + + coll := m.database.C(resColl) + coll.DropCollection() + for _, book := range books { + err = coll.Insert(book) + if err != nil { + return err + } + } + return nil } func (m *MR) UpdateHourVisits(statsColl *mgo.Collection) error { diff --git a/trantor.go b/trantor.go index 6a1ed8f..36560ce 100644 --- a/trantor.go +++ b/trantor.go @@ -130,9 +130,9 @@ func indexHandler(h handler) { data.Tags, _ = h.db.GetTags(TAGS_DISPLAY) data.S = GetStatus(h) data.S.Home = true - data.Books, data.Count, _ = h.db.GetBooks(bson.M{"active": true}, 6) - data.VisitedBooks, _ = h.db.GetVisitedBooks(6) - data.DownloadedBooks, _ = h.db.GetDownloadedBooks(6) + data.Books, data.Count, _ = h.db.GetBooks(bson.M{"active": true}, BOOKS_FRONT_PAGE) + data.VisitedBooks, _ = h.db.GetVisitedBooks(BOOKS_FRONT_PAGE) + data.DownloadedBooks, _ = h.db.GetDownloadedBooks(BOOKS_FRONT_PAGE) data.News = getNews(1, DAYS_NEWS_INDEXPAGE, h.db) loadTemplate(h.w, "index", data) } From 8927a8cd6dcbda7ad2ff1c266267670817c6f085 Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Tue, 18 Feb 2014 22:16:38 +0100 Subject: [PATCH 09/11] Clean up code --- database.go | 79 +++++++++++++----------- mapreduce.go => db_stats.go | 118 +++++++++++++----------------------- trantor.go | 6 +- 3 files changed, 89 insertions(+), 114 deletions(-) rename mapreduce.go => db_stats.go (51%) diff --git a/database.go b/database.go index 29b4fbb..36c6834 100644 --- a/database.go +++ b/database.go @@ -176,14 +176,14 @@ func (d *DB) GetBooks(query bson.M, r ...int) (books []Book, num int, err error) /* Get the most visited books */ -func (d *DB) GetVisitedBooks(num int) (books []Book, err error) { +func (d *DB) GetVisitedBooks() (books []Book, err error) { visitedColl := d.session.DB(DB_NAME).C(VISITED_COLL) - bookId, err := GetBooksVisited(num, visitedColl) + bookId, err := GetBooksVisited(visitedColl) if err != nil { return nil, err } - books = make([]Book, num) + books = make([]Book, len(bookId)) for i, id := range bookId { booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL) booksColl.Find(bson.M{"_id": id}).One(&books[i]) @@ -193,21 +193,22 @@ func (d *DB) GetVisitedBooks(num int) (books []Book, err error) { } func (d *DB) UpdateMostVisited() error { - statsColl := d.session.DB(DB_NAME).C(STATS_COLL) - mr := NewMR(d.session.DB(DB_NAME)) - return mr.UpdateMostVisited(statsColl) + var u DBUpdate + u.src = d.session.DB(DB_NAME).C(STATS_COLL) + u.dst = d.session.DB(DB_NAME).C(VISITED_COLL) + return u.UpdateMostBooks("book") } /* Get the most downloaded books */ -func (d *DB) GetDownloadedBooks(num int) (books []Book, err error) { +func (d *DB) GetDownloadedBooks() (books []Book, err error) { downloadedColl := d.session.DB(DB_NAME).C(DOWNLOADED_COLL) - bookId, err := GetBooksVisited(num, downloadedColl) + bookId, err := GetBooksVisited(downloadedColl) if err != nil { return nil, err } - books = make([]Book, num) + books = make([]Book, len(bookId)) for i, id := range bookId { booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL) booksColl.Find(bson.M{"_id": id}).One(&books[i]) @@ -217,9 +218,10 @@ func (d *DB) GetDownloadedBooks(num int) (books []Book, err error) { } func (d *DB) UpdateDownloadedBooks() error { - statsColl := d.session.DB(DB_NAME).C(STATS_COLL) - mr := NewMR(d.session.DB(DB_NAME)) - return mr.UpdateMostDownloaded(statsColl) + var u DBUpdate + u.src = d.session.DB(DB_NAME).C(STATS_COLL) + u.dst = d.session.DB(DB_NAME).C(DOWNLOADED_COLL) + return u.UpdateMostBooks("download") } /* optional parameters: length and start index @@ -244,15 +246,16 @@ func (d *DB) GetFS(prefix string) *mgo.GridFS { return d.session.DB(DB_NAME).GridFS(prefix) } -func (d *DB) GetTags(numTags int) ([]string, error) { +func (d *DB) GetTags() ([]string, error) { tagsColl := d.session.DB(DB_NAME).C(TAGS_COLL) - return GetTags(numTags, tagsColl) + return GetTags(tagsColl) } func (d *DB) UpdateTags() error { - booksColl := d.session.DB(DB_NAME).C(BOOKS_COLL) - mr := NewMR(d.session.DB(DB_NAME)) - return mr.UpdateTags(booksColl) + var u DBUpdate + u.src = d.session.DB(DB_NAME).C(BOOKS_COLL) + u.dst = d.session.DB(DB_NAME).C(TAGS_COLL) + return u.UpdateTags() } type Visits struct { @@ -266,9 +269,10 @@ func (d *DB) GetHourVisits() ([]Visits, error) { } func (d *DB) UpdateHourVisits() error { - statsColl := d.session.DB(DB_NAME).C(STATS_COLL) - mr := NewMR(d.session.DB(DB_NAME)) - return mr.UpdateHourVisits(statsColl) + var u DBUpdate + u.src = d.session.DB(DB_NAME).C(STATS_COLL) + u.dst = d.session.DB(DB_NAME).C(HOURLY_VISITS_COLL) + return u.UpdateHourVisits() } func (d *DB) GetDayVisits() ([]Visits, error) { @@ -277,9 +281,10 @@ func (d *DB) GetDayVisits() ([]Visits, error) { } func (d *DB) UpdateDayVisits() error { - statsColl := d.session.DB(DB_NAME).C(STATS_COLL) - mr := NewMR(d.session.DB(DB_NAME)) - return mr.UpdateDayVisits(statsColl) + var u DBUpdate + u.src = d.session.DB(DB_NAME).C(STATS_COLL) + u.dst = d.session.DB(DB_NAME).C(DAILY_VISITS_COLL) + return u.UpdateDayVisits() } func (d *DB) GetMonthVisits() ([]Visits, error) { @@ -288,9 +293,10 @@ func (d *DB) GetMonthVisits() ([]Visits, error) { } func (d *DB) UpdateMonthVisits() error { - statsColl := d.session.DB(DB_NAME).C(STATS_COLL) - mr := NewMR(d.session.DB(DB_NAME)) - return mr.UpdateMonthVisits(statsColl) + var u DBUpdate + u.src = d.session.DB(DB_NAME).C(STATS_COLL) + u.dst = d.session.DB(DB_NAME).C(MONTHLY_VISITS_COLL) + return u.UpdateMonthVisits() } func (d *DB) GetHourDownloads() ([]Visits, error) { @@ -299,9 +305,10 @@ func (d *DB) GetHourDownloads() ([]Visits, error) { } func (d *DB) UpdateHourDownloads() error { - statsColl := d.session.DB(DB_NAME).C(STATS_COLL) - mr := NewMR(d.session.DB(DB_NAME)) - return mr.UpdateHourDownloads(statsColl) + var u DBUpdate + u.src = d.session.DB(DB_NAME).C(STATS_COLL) + u.dst = d.session.DB(DB_NAME).C(HOURLY_DOWNLOADS_COLL) + return u.UpdateHourDownloads() } func (d *DB) GetDayDownloads() ([]Visits, error) { @@ -310,9 +317,10 @@ func (d *DB) GetDayDownloads() ([]Visits, error) { } func (d *DB) UpdateDayDownloads() error { - statsColl := d.session.DB(DB_NAME).C(STATS_COLL) - mr := NewMR(d.session.DB(DB_NAME)) - return mr.UpdateDayDownloads(statsColl) + var u DBUpdate + u.src = d.session.DB(DB_NAME).C(STATS_COLL) + u.dst = d.session.DB(DB_NAME).C(DAILY_DOWNLOADS_COLL) + return u.UpdateDayDownloads() } func (d *DB) GetMonthDownloads() ([]Visits, error) { @@ -321,7 +329,8 @@ func (d *DB) GetMonthDownloads() ([]Visits, error) { } func (d *DB) UpdateMonthDownloads() error { - statsColl := d.session.DB(DB_NAME).C(STATS_COLL) - mr := NewMR(d.session.DB(DB_NAME)) - return mr.UpdateMonthDownloads(statsColl) + var u DBUpdate + u.src = d.session.DB(DB_NAME).C(STATS_COLL) + u.dst = d.session.DB(DB_NAME).C(MONTHLY_DOWNLOADS_COLL) + return u.UpdateMonthDownloads() } diff --git a/mapreduce.go b/db_stats.go similarity index 51% rename from mapreduce.go rename to db_stats.go index 83ed2f9..1394e81 100644 --- a/mapreduce.go +++ b/db_stats.go @@ -6,11 +6,16 @@ import ( "time" ) -func GetTags(numTags int, tagsColl *mgo.Collection) ([]string, error) { +type DBUpdate struct { + src *mgo.Collection + dst *mgo.Collection +} + +func GetTags(tagsColl *mgo.Collection) ([]string, error) { var result []struct { Tag string "_id" } - err := tagsColl.Find(nil).Sort("-count").Limit(numTags).All(&result) + err := tagsColl.Find(nil).Sort("-count").All(&result) if err != nil { return nil, err } @@ -22,11 +27,11 @@ func GetTags(numTags int, tagsColl *mgo.Collection) ([]string, error) { return tags, nil } -func GetBooksVisited(num int, visitedColl *mgo.Collection) ([]bson.ObjectId, error) { +func GetBooksVisited(visitedColl *mgo.Collection) ([]bson.ObjectId, error) { var result []struct { Book bson.ObjectId "_id" } - err := visitedColl.Find(nil).Sort("-count").Limit(num).All(&result) + err := visitedColl.Find(nil).Sort("-count").All(&result) if err != nil { return nil, err } @@ -44,22 +49,12 @@ func GetVisits(visitsColl *mgo.Collection) ([]Visits, error) { return result, err } -type MR struct { - database *mgo.Database -} - -func NewMR(database *mgo.Database) *MR { - m := new(MR) - m.database = database - return m -} - -func (m *MR) UpdateTags(booksColl *mgo.Collection) error { +func (u *DBUpdate) UpdateTags() error { var tags []struct { Tag string "_id" Count int "count" } - err := booksColl.Pipe([]bson.M{ + err := u.src.Pipe([]bson.M{ {"$project": bson.M{"subject": 1}}, {"$unwind": "$subject"}, {"$group": bson.M{"_id": "$subject", "count": bson.M{"$sum": 1}}}, @@ -70,10 +65,9 @@ func (m *MR) UpdateTags(booksColl *mgo.Collection) error { return err } - tagsColl := m.database.C(TAGS_COLL) - tagsColl.DropCollection() + u.dst.DropCollection() for _, tag := range tags { - err = tagsColl.Insert(tag) + err = u.dst.Insert(tag) if err != nil { return err } @@ -81,15 +75,7 @@ func (m *MR) UpdateTags(booksColl *mgo.Collection) error { return nil } -func (m *MR) UpdateMostVisited(statsColl *mgo.Collection) error { - return m.updateMostBooks(statsColl, "book", VISITED_COLL) -} - -func (m *MR) UpdateMostDownloaded(statsColl *mgo.Collection) error { - return m.updateMostBooks(statsColl, "download", DOWNLOADED_COLL) -} - -func (m *MR) updateMostBooks(statsColl *mgo.Collection, section string, resColl string) error { +func (u *DBUpdate) UpdateMostBooks(section string) error { const numDays = 30 start := time.Now().UTC().Add(-numDays * 24 * time.Hour) @@ -97,7 +83,7 @@ func (m *MR) updateMostBooks(statsColl *mgo.Collection, section string, resColl Book string "_id" Count int "count" } - err := statsColl.Pipe([]bson.M{ + err := u.src.Pipe([]bson.M{ {"$match": bson.M{"date": bson.M{"$gt": start}, "section": section}}, {"$project": bson.M{"id": 1}}, {"$group": bson.M{"_id": "$id", "count": bson.M{"$sum": 1}}}, @@ -108,10 +94,9 @@ func (m *MR) updateMostBooks(statsColl *mgo.Collection, section string, resColl return err } - coll := m.database.C(resColl) - coll.DropCollection() + u.dst.DropCollection() for _, book := range books { - err = coll.Insert(book) + err = u.dst.Insert(book) if err != nil { return err } @@ -119,84 +104,83 @@ func (m *MR) updateMostBooks(statsColl *mgo.Collection, section string, resColl return nil } -func (m *MR) UpdateHourVisits(statsColl *mgo.Collection) error { +func (u *DBUpdate) UpdateHourVisits() error { f := func(t time.Time) time.Time { const span = time.Hour return t.Add(span).Truncate(span) } const numDays = 2 spanStore := numDays * 24 * time.Hour - return m.updateVisits(f, spanStore, HOURLY_VISITS_COLL, true) + return u.updateVisits(f, spanStore, HOURLY_VISITS_COLL, true) } -func (m *MR) UpdateDayVisits(statsColl *mgo.Collection) error { +func (u *DBUpdate) UpdateDayVisits() error { f := func(t time.Time) time.Time { const span = 24 * time.Hour return t.Add(span).Truncate(span) } const numDays = 30 spanStore := numDays * 24 * time.Hour - return m.updateVisits(f, spanStore, DAILY_VISITS_COLL, true) + return u.updateVisits(f, spanStore, DAILY_VISITS_COLL, true) } -func (m *MR) UpdateMonthVisits(statsColl *mgo.Collection) error { +func (u *DBUpdate) UpdateMonthVisits() error { f := func(t time.Time) time.Time { const span = 24 * time.Hour return t.AddDate(0, 1, 1-t.Day()).Truncate(span) } const numDays = 365 spanStore := numDays * 24 * time.Hour - return m.updateVisits(f, spanStore, MONTHLY_VISITS_COLL, true) + return u.updateVisits(f, spanStore, MONTHLY_VISITS_COLL, true) } -func (m *MR) UpdateHourDownloads(statsColl *mgo.Collection) error { +func (u *DBUpdate) UpdateHourDownloads() error { f := func(t time.Time) time.Time { const span = time.Hour return t.Add(span).Truncate(span) } const numDays = 2 spanStore := numDays * 24 * time.Hour - return m.updateVisits(f, spanStore, HOURLY_DOWNLOADS_COLL, false) + return u.updateVisits(f, spanStore, HOURLY_DOWNLOADS_COLL, false) } -func (m *MR) UpdateDayDownloads(statsColl *mgo.Collection) error { +func (u *DBUpdate) UpdateDayDownloads() error { f := func(t time.Time) time.Time { const span = 24 * time.Hour return t.Add(span).Truncate(span) } const numDays = 30 spanStore := numDays * 24 * time.Hour - return m.updateVisits(f, spanStore, DAILY_DOWNLOADS_COLL, false) + return u.updateVisits(f, spanStore, DAILY_DOWNLOADS_COLL, false) } -func (m *MR) UpdateMonthDownloads(statsColl *mgo.Collection) error { +func (u *DBUpdate) UpdateMonthDownloads() error { f := func(t time.Time) time.Time { const span = 24 * time.Hour return t.AddDate(0, 1, 1-t.Day()).Truncate(span) } const numDays = 365 spanStore := numDays * 24 * time.Hour - return m.updateVisits(f, spanStore, MONTHLY_DOWNLOADS_COLL, false) + return u.updateVisits(f, spanStore, MONTHLY_DOWNLOADS_COLL, false) } -func (m *MR) updateVisits(incTime func(time.Time) time.Time, spanStore time.Duration, coll string, useSession bool) error { - storeColl := m.database.C(coll) - start := m.calculateStart(spanStore, storeColl) +func (u *DBUpdate) updateVisits(incTime func(time.Time) time.Time, spanStore time.Duration, coll string, useSession bool) error { + start := u.calculateStart(spanStore) for start.Before(time.Now().UTC()) { stop := incTime(start) var count int var err error if useSession { - count = m.countVisits(start, stop) + count = u.countVisits(start, stop) } else { - count, err = m.countDownloads(start, stop) + count, err = u.countDownloads(start, stop) } if err != nil { return err } - err = storeColl.Insert(bson.M{"date": start, "count": count}) + err = u.dst.Insert(bson.M{"date": start, "count": count}) if err != nil { return err } @@ -204,29 +188,28 @@ func (m *MR) updateVisits(incTime func(time.Time) time.Time, spanStore time.Dura start = stop } - _, err := storeColl.RemoveAll(bson.M{"date": bson.M{"$lt": time.Now().UTC().Add(-spanStore)}}) + _, err := u.dst.RemoveAll(bson.M{"date": bson.M{"$lt": time.Now().UTC().Add(-spanStore)}}) return err } -func (m *MR) calculateStart(spanStore time.Duration, storeColl *mgo.Collection) time.Time { +func (u *DBUpdate) calculateStart(spanStore time.Duration) time.Time { var date struct { Id bson.ObjectId `bson:"_id"` Date time.Time `bson:"date"` } - err := storeColl.Find(bson.M{}).Sort("-date").One(&date) + err := u.dst.Find(bson.M{}).Sort("-date").One(&date) if err == nil { - storeColl.RemoveId(date.Id) + u.dst.RemoveId(date.Id) return date.Date } return time.Now().UTC().Add(-spanStore).Truncate(time.Hour) } -func (m *MR) countVisits(start time.Time, stop time.Time) int { - statsColl := m.database.C(STATS_COLL) +func (u *DBUpdate) countVisits(start time.Time, stop time.Time) int { var result struct { Count int "count" } - err := statsColl.Pipe([]bson.M{ + err := u.src.Pipe([]bson.M{ {"$match": bson.M{"date": bson.M{"$gte": start, "$lt": stop}}}, {"$group": bson.M{"_id": "$session"}}, {"$group": bson.M{"_id": 1, "count": bson.M{"$sum": 1}}}, @@ -238,24 +221,7 @@ func (m *MR) countVisits(start time.Time, stop time.Time) int { return result.Count } -func (m *MR) countDownloads(start time.Time, stop time.Time) (int, error) { +func (u *DBUpdate) countDownloads(start time.Time, stop time.Time) (int, error) { query := bson.M{"date": bson.M{"$gte": start, "$lt": stop}, "section": "download"} - statsColl := m.database.C(STATS_COLL) - return statsColl.Find(query).Count() -} - -func (m *MR) update(mr *mgo.MapReduce, query bson.M, queryColl *mgo.Collection, storeColl string) error { - metaColl := m.database.C(META_COLL) - _, err := metaColl.RemoveAll(bson.M{"type": storeColl}) - if err != nil { - return err - } - - mr.Out = bson.M{"replace": storeColl} - _, err = queryColl.Find(query).MapReduce(mr, nil) - if err != nil { - return err - } - - return metaColl.Insert(bson.M{"type": storeColl}) + return u.src.Find(query).Count() } diff --git a/trantor.go b/trantor.go index 36560ce..a226b74 100644 --- a/trantor.go +++ b/trantor.go @@ -127,12 +127,12 @@ type indexData struct { func indexHandler(h handler) { var data indexData - data.Tags, _ = h.db.GetTags(TAGS_DISPLAY) + data.Tags, _ = h.db.GetTags() data.S = GetStatus(h) data.S.Home = true data.Books, data.Count, _ = h.db.GetBooks(bson.M{"active": true}, BOOKS_FRONT_PAGE) - data.VisitedBooks, _ = h.db.GetVisitedBooks(BOOKS_FRONT_PAGE) - data.DownloadedBooks, _ = h.db.GetDownloadedBooks(BOOKS_FRONT_PAGE) + data.VisitedBooks, _ = h.db.GetVisitedBooks() + data.DownloadedBooks, _ = h.db.GetDownloadedBooks() data.News = getNews(1, DAYS_NEWS_INDEXPAGE, h.db) loadTemplate(h.w, "index", data) } From cdc25f1094a366df40988430c56a24d73282d85b Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Wed, 19 Feb 2014 00:59:55 +0100 Subject: [PATCH 10/11] Clean up db stats visits --- database.go | 59 +++++++++++++++++---------------- stats.go | 95 ++++++++++------------------------------------------- 2 files changed, 48 insertions(+), 106 deletions(-) diff --git a/database.go b/database.go index 36c6834..8f2beb5 100644 --- a/database.go +++ b/database.go @@ -4,6 +4,7 @@ import log "github.com/cihub/seelog" import ( "crypto/md5" + "errors" "labix.org/v2/mgo" "labix.org/v2/mgo/bson" "os" @@ -258,14 +259,41 @@ func (d *DB) UpdateTags() error { return u.UpdateTags() } +type VisitType int + +const ( + hourly_visits = iota + daily_visits + monthly_visits + hourly_downloads + daily_downloads + monthly_downloads +) + type Visits struct { Date time.Time "date" Count int "count" } -func (d *DB) GetHourVisits() ([]Visits, error) { - hourlyColl := d.session.DB(DB_NAME).C(HOURLY_VISITS_COLL) - return GetVisits(hourlyColl) +func (d *DB) GetVisits(visitType VisitType) ([]Visits, error) { + var coll *mgo.Collection + switch visitType { + case hourly_visits: + coll = d.session.DB(DB_NAME).C(HOURLY_VISITS_COLL) + case daily_visits: + coll = d.session.DB(DB_NAME).C(DAILY_VISITS_COLL) + case monthly_visits: + coll = d.session.DB(DB_NAME).C(MONTHLY_VISITS_COLL) + case hourly_downloads: + coll = d.session.DB(DB_NAME).C(HOURLY_DOWNLOADS_COLL) + case daily_downloads: + coll = d.session.DB(DB_NAME).C(DAILY_DOWNLOADS_COLL) + case monthly_downloads: + coll = d.session.DB(DB_NAME).C(MONTHLY_DOWNLOADS_COLL) + default: + return nil, errors.New("Not valid VisitType") + } + return GetVisits(coll) } func (d *DB) UpdateHourVisits() error { @@ -275,11 +303,6 @@ func (d *DB) UpdateHourVisits() error { return u.UpdateHourVisits() } -func (d *DB) GetDayVisits() ([]Visits, error) { - dailyColl := d.session.DB(DB_NAME).C(DAILY_VISITS_COLL) - return GetVisits(dailyColl) -} - func (d *DB) UpdateDayVisits() error { var u DBUpdate u.src = d.session.DB(DB_NAME).C(STATS_COLL) @@ -287,11 +310,6 @@ func (d *DB) UpdateDayVisits() error { return u.UpdateDayVisits() } -func (d *DB) GetMonthVisits() ([]Visits, error) { - monthlyColl := d.session.DB(DB_NAME).C(MONTHLY_VISITS_COLL) - return GetVisits(monthlyColl) -} - func (d *DB) UpdateMonthVisits() error { var u DBUpdate u.src = d.session.DB(DB_NAME).C(STATS_COLL) @@ -299,11 +317,6 @@ func (d *DB) UpdateMonthVisits() error { return u.UpdateMonthVisits() } -func (d *DB) GetHourDownloads() ([]Visits, error) { - hourlyColl := d.session.DB(DB_NAME).C(HOURLY_DOWNLOADS_COLL) - return GetVisits(hourlyColl) -} - func (d *DB) UpdateHourDownloads() error { var u DBUpdate u.src = d.session.DB(DB_NAME).C(STATS_COLL) @@ -311,11 +324,6 @@ func (d *DB) UpdateHourDownloads() error { return u.UpdateHourDownloads() } -func (d *DB) GetDayDownloads() ([]Visits, error) { - dailyColl := d.session.DB(DB_NAME).C(DAILY_DOWNLOADS_COLL) - return GetVisits(dailyColl) -} - func (d *DB) UpdateDayDownloads() error { var u DBUpdate u.src = d.session.DB(DB_NAME).C(STATS_COLL) @@ -323,11 +331,6 @@ func (d *DB) UpdateDayDownloads() error { return u.UpdateDayDownloads() } -func (d *DB) GetMonthDownloads() ([]Visits, error) { - monthlyColl := d.session.DB(DB_NAME).C(MONTHLY_DOWNLOADS_COLL) - return GetVisits(monthlyColl) -} - func (d *DB) UpdateMonthDownloads() error { var u DBUpdate u.src = d.session.DB(DB_NAME).C(STATS_COLL) diff --git a/stats.go b/stats.go index 626aafa..d6e48cd 100644 --- a/stats.go +++ b/stats.go @@ -69,12 +69,12 @@ func statsHandler(h handler) { var data statsData data.S = GetStatus(h) data.S.Stats = true - data.HVisits = getHourlyVisits(h.db) - data.DVisits = getDailyVisits(h.db) - data.MVisits = getMonthlyVisits(h.db) - data.HDownloads = getHourlyDownloads(h.db) - data.DDownloads = getDailyDownloads(h.db) - data.MDownloads = getMonthlyDownloads(h.db) + data.HVisits = getVisits(hourlyLabel, h.db, hourly_visits) + data.DVisits = getVisits(dailyLabel, h.db, daily_visits) + data.MVisits = getVisits(monthlyLabel, h.db, monthly_visits) + data.HDownloads = getVisits(hourlyLabel, h.db, hourly_downloads) + data.DDownloads = getVisits(dailyLabel, h.db, daily_downloads) + data.MDownloads = getVisits(monthlyLabel, h.db, monthly_downloads) loadTemplate(h.w, "stats", data) } @@ -94,89 +94,28 @@ type visitData struct { Count int } -func getHourlyVisits(db *DB) []visitData { - var visits []visitData - - visit, _ := db.GetHourVisits() - for _, v := range visit { - var elem visitData - hour := v.Date.UTC().Hour() - elem.Label = strconv.Itoa(hour + 1) - elem.Count = v.Count - visits = append(visits, elem) - } - - return visits +func hourlyLabel(date time.Time) string { + return strconv.Itoa(date.Hour() + 1) } -func getDailyVisits(db *DB) []visitData { - var visits []visitData - - visit, _ := db.GetDayVisits() - for _, v := range visit { - var elem visitData - day := v.Date.UTC().Day() - elem.Label = strconv.Itoa(day) - elem.Count = v.Count - visits = append(visits, elem) - } - - return visits +func dailyLabel(date time.Time) string { + return strconv.Itoa(date.Day()) } -func getMonthlyVisits(db *DB) []visitData { - var visits []visitData - - visit, _ := db.GetMonthVisits() - for _, v := range visit { - var elem visitData - month := v.Date.UTC().Month() - elem.Label = month.String() - elem.Count = v.Count - visits = append(visits, elem) - } - - return visits +func monthlyLabel(date time.Time) string { + return date.Month().String() } -func getHourlyDownloads(db *DB) []visitData { +func getVisits(funcLabel func(time.Time) string, db *DB, visitType VisitType) []visitData { var visits []visitData - visit, _ := db.GetHourDownloads() - for _, v := range visit { - var elem visitData - hour := v.Date.UTC().Hour() - elem.Label = strconv.Itoa(hour + 1) - elem.Count = v.Count - visits = append(visits, elem) + visit, err := db.GetVisits(visitType) + if err != nil { + log.Warn("GetVisits error (", visitType, "): ", err) } - - return visits -} - -func getDailyDownloads(db *DB) []visitData { - var visits []visitData - - visit, _ := db.GetDayDownloads() for _, v := range visit { var elem visitData - day := v.Date.UTC().Day() - elem.Label = strconv.Itoa(day) - elem.Count = v.Count - visits = append(visits, elem) - } - - return visits -} - -func getMonthlyDownloads(db *DB) []visitData { - var visits []visitData - - visit, _ := db.GetMonthDownloads() - for _, v := range visit { - var elem visitData - month := v.Date.UTC().Month() - elem.Label = month.String() + elem.Label = funcLabel(v.Date.UTC()) elem.Count = v.Count visits = append(visits, elem) } From e67e23c0507036958cc4e9975d0ac51ecc3c9fd8 Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Wed, 19 Feb 2014 01:20:10 +0100 Subject: [PATCH 11/11] Clean up visit updater --- database.go | 12 +++++----- db_stats.go | 65 ++++++++++++++++------------------------------------- 2 files changed, 25 insertions(+), 52 deletions(-) diff --git a/database.go b/database.go index 8f2beb5..bb0bbed 100644 --- a/database.go +++ b/database.go @@ -300,40 +300,40 @@ func (d *DB) UpdateHourVisits() error { var u DBUpdate u.src = d.session.DB(DB_NAME).C(STATS_COLL) u.dst = d.session.DB(DB_NAME).C(HOURLY_VISITS_COLL) - return u.UpdateHourVisits() + return u.UpdateHourVisits(false) } func (d *DB) UpdateDayVisits() error { var u DBUpdate u.src = d.session.DB(DB_NAME).C(STATS_COLL) u.dst = d.session.DB(DB_NAME).C(DAILY_VISITS_COLL) - return u.UpdateDayVisits() + return u.UpdateDayVisits(false) } func (d *DB) UpdateMonthVisits() error { var u DBUpdate u.src = d.session.DB(DB_NAME).C(STATS_COLL) u.dst = d.session.DB(DB_NAME).C(MONTHLY_VISITS_COLL) - return u.UpdateMonthVisits() + return u.UpdateMonthVisits(false) } func (d *DB) UpdateHourDownloads() error { var u DBUpdate u.src = d.session.DB(DB_NAME).C(STATS_COLL) u.dst = d.session.DB(DB_NAME).C(HOURLY_DOWNLOADS_COLL) - return u.UpdateHourDownloads() + return u.UpdateHourVisits(true) } func (d *DB) UpdateDayDownloads() error { var u DBUpdate u.src = d.session.DB(DB_NAME).C(STATS_COLL) u.dst = d.session.DB(DB_NAME).C(DAILY_DOWNLOADS_COLL) - return u.UpdateDayDownloads() + return u.UpdateDayVisits(true) } func (d *DB) UpdateMonthDownloads() error { var u DBUpdate u.src = d.session.DB(DB_NAME).C(STATS_COLL) u.dst = d.session.DB(DB_NAME).C(MONTHLY_DOWNLOADS_COLL) - return u.UpdateMonthDownloads() + return u.UpdateMonthVisits(true) } diff --git a/db_stats.go b/db_stats.go index 1394e81..986cdf6 100644 --- a/db_stats.go +++ b/db_stats.go @@ -104,77 +104,50 @@ func (u *DBUpdate) UpdateMostBooks(section string) error { return nil } -func (u *DBUpdate) UpdateHourVisits() error { - f := func(t time.Time) time.Time { - const span = time.Hour - return t.Add(span).Truncate(span) - } +func (u *DBUpdate) UpdateHourVisits(isDownloads bool) error { const numDays = 2 spanStore := numDays * 24 * time.Hour - return u.updateVisits(f, spanStore, HOURLY_VISITS_COLL, true) + return u.updateVisits(hourInc, spanStore, isDownloads) } -func (u *DBUpdate) UpdateDayVisits() error { - f := func(t time.Time) time.Time { - const span = 24 * time.Hour - return t.Add(span).Truncate(span) - } +func (u *DBUpdate) UpdateDayVisits(isDownloads bool) error { const numDays = 30 spanStore := numDays * 24 * time.Hour - return u.updateVisits(f, spanStore, DAILY_VISITS_COLL, true) + return u.updateVisits(dayInc, spanStore, isDownloads) } -func (u *DBUpdate) UpdateMonthVisits() error { - f := func(t time.Time) time.Time { - const span = 24 * time.Hour - return t.AddDate(0, 1, 1-t.Day()).Truncate(span) - } +func (u *DBUpdate) UpdateMonthVisits(isDownloads bool) error { const numDays = 365 spanStore := numDays * 24 * time.Hour - return u.updateVisits(f, spanStore, MONTHLY_VISITS_COLL, true) + return u.updateVisits(monthInc, spanStore, isDownloads) } -func (u *DBUpdate) UpdateHourDownloads() error { - f := func(t time.Time) time.Time { - const span = time.Hour - return t.Add(span).Truncate(span) - } - const numDays = 2 - spanStore := numDays * 24 * time.Hour - return u.updateVisits(f, spanStore, HOURLY_DOWNLOADS_COLL, false) +func hourInc(date time.Time) time.Time { + const span = time.Hour + return date.Add(span).Truncate(span) } -func (u *DBUpdate) UpdateDayDownloads() error { - f := func(t time.Time) time.Time { - const span = 24 * time.Hour - return t.Add(span).Truncate(span) - } - const numDays = 30 - spanStore := numDays * 24 * time.Hour - return u.updateVisits(f, spanStore, DAILY_DOWNLOADS_COLL, false) +func dayInc(date time.Time) time.Time { + const span = 24 * time.Hour + return date.Add(span).Truncate(span) } -func (u *DBUpdate) UpdateMonthDownloads() error { - f := func(t time.Time) time.Time { - const span = 24 * time.Hour - return t.AddDate(0, 1, 1-t.Day()).Truncate(span) - } - const numDays = 365 - spanStore := numDays * 24 * time.Hour - return u.updateVisits(f, spanStore, MONTHLY_DOWNLOADS_COLL, false) +func monthInc(date time.Time) time.Time { + const span = 24 * time.Hour + return date.AddDate(0, 1, 1-date.Day()).Truncate(span) } -func (u *DBUpdate) updateVisits(incTime func(time.Time) time.Time, spanStore time.Duration, coll string, useSession bool) error { +func (u *DBUpdate) updateVisits(incTime func(time.Time) time.Time, spanStore time.Duration, isDownloads bool) error { start := u.calculateStart(spanStore) for start.Before(time.Now().UTC()) { stop := incTime(start) var count int var err error - if useSession { - count = u.countVisits(start, stop) - } else { + if isDownloads { count, err = u.countDownloads(start, stop) + } else { + count = u.countVisits(start, stop) } if err != nil { return err