From a23264ec8ddb516a24fad50ae6f346f07e730b2b Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Tue, 24 Sep 2013 14:22:53 +0200 Subject: [PATCH] Add a downloads collumn to the statistics --- config.go | 33 ++++++------ database.go | 18 +++++++ mapreduce.go | 119 ++++++++++++++++++++++++++++++++++++++++++- stats.go | 71 +++++++++++++++++++++++--- templates/stats.html | 45 +++++++++++----- 5 files changed, 251 insertions(+), 35 deletions(-) diff --git a/config.go b/config.go index f92f3a3..67fe58c 100644 --- a/config.go +++ b/config.go @@ -3,21 +3,24 @@ package main const ( PORT = "8080" - DB_IP = "127.0.0.1" - DB_NAME = "trantor" - META_COLL = "meta" - BOOKS_COLL = "books" - TAGS_COLL = "tags" - VISITED_COLL = "visited" - DOWNLOADED_COLL = "downloaded" - HOURLY_VISITS_COLL = "visits.hourly" - DAILY_VISITS_COLL = "visits.daily" - MONTHLY_VISITS_COLL = "visits.monthly" - USERS_COLL = "users" - NEWS_COLL = "news" - STATS_COLL = "statistics" - FS_BOOKS = "fs_books" - FS_IMGS = "fs_imgs" + DB_IP = "127.0.0.1" + DB_NAME = "trantor" + META_COLL = "meta" + BOOKS_COLL = "books" + TAGS_COLL = "tags" + VISITED_COLL = "visited" + DOWNLOADED_COLL = "downloaded" + HOURLY_VISITS_COLL = "visits.hourly" + DAILY_VISITS_COLL = "visits.daily" + MONTHLY_VISITS_COLL = "visits.monthly" + HOURLY_DOWNLOADS_COLL = "downloads.hourly" + DAILY_DOWNLOADS_COLL = "downloads.daily" + MONTHLY_DOWNLOADS_COLL = "downloads.monthly" + USERS_COLL = "users" + NEWS_COLL = "news" + STATS_COLL = "statistics" + FS_BOOKS = "fs_books" + FS_IMGS = "fs_imgs" PASS_SALT = "ImperialLibSalt" MINUTES_UPDATE_TAGS = 11 diff --git a/database.go b/database.go index ea1b65d..3e5f5ab 100644 --- a/database.go +++ b/database.go @@ -258,3 +258,21 @@ func (d *DB) GetMonthVisits(start time.Time) ([]Visits, error) { mr := NewMR(d.session.DB(DB_NAME)) return mr.GetMonthVisits(start, statsColl) } + +func (d *DB) GetHourDownloads(start time.Time) ([]Visits, error) { + statsColl := d.session.DB(DB_NAME).C(STATS_COLL) + mr := NewMR(d.session.DB(DB_NAME)) + return mr.GetHourDownloads(start, statsColl) +} + +func (d *DB) GetDayDownloads(start time.Time) ([]Visits, error) { + statsColl := d.session.DB(DB_NAME).C(STATS_COLL) + mr := NewMR(d.session.DB(DB_NAME)) + return mr.GetDayDowloads(start, statsColl) +} + +func (d *DB) GetMonthDownloads(start time.Time) ([]Visits, error) { + statsColl := d.session.DB(DB_NAME).C(STATS_COLL) + mr := NewMR(d.session.DB(DB_NAME)) + return mr.GetMonthDowloads(start, statsColl) +} diff --git a/mapreduce.go b/mapreduce.go index 4f4b55c..8b097ae 100644 --- a/mapreduce.go +++ b/mapreduce.go @@ -129,7 +129,7 @@ func (m *MR) GetHourVisits(start time.Time, statsColl *mgo.Collection) ([]Visits var date = Date.UTC(this.date.getUTCFullYear(), this.date.getUTCMonth(), this.date.getUTCDate(), - this.date.getUTCHours()); + this.date.getUTCHours()); emit({date: date, session: this.session}, 1); }` mr.Reduce = reduce @@ -228,6 +228,123 @@ func (m *MR) GetMonthVisits(start time.Time, statsColl *mgo.Collection) ([]Visit return result, err } +func (m *MR) GetHourDownloads(start time.Time, statsColl *mgo.Collection) ([]Visits, error) { + if m.isOutdated(HOURLY_DOWNLOADS_COLL, MINUTES_UPDATE_HOURLY) { + const reduce = `function(date, vals) { + var count = 0; + vals.forEach(function(v) { count += v; }); + return count; + }` + var mr mgo.MapReduce + mr.Map = `function() { + if (this.section == "download") { + var date = Date.UTC(this.date.getUTCFullYear(), + this.date.getUTCMonth(), + this.date.getUTCDate(), + this.date.getUTCHours()); + emit({date: date}, 1); + } + }` + mr.Reduce = reduce + err := m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, HOURLY_DOWNLOADS_COLL+"_raw") + if err != nil { + return nil, err + } + var mr2 mgo.MapReduce + mr2.Map = `function() { + emit(this['_id']['date'], 1); + }` + mr2.Reduce = reduce + hourly_raw := m.database.C(HOURLY_DOWNLOADS_COLL + "_raw") + err = m.update(&mr2, bson.M{}, hourly_raw, HOURLY_DOWNLOADS_COLL) + if err != nil { + return nil, err + } + } + + var result []Visits + hourlyColl := m.database.C(HOURLY_DOWNLOADS_COLL) + err := hourlyColl.Find(nil).All(&result) + return result, err +} + +func (m *MR) GetDayDowloads(start time.Time, statsColl *mgo.Collection) ([]Visits, error) { + if m.isOutdated(DAILY_DOWNLOADS_COLL, MINUTES_UPDATE_DAILY) { + const reduce = `function(date, vals) { + var count = 0; + vals.forEach(function(v) { count += v; }); + return count; + }` + var mr mgo.MapReduce + mr.Map = `function() { + if (this.section == "download") { + var date = Date.UTC(this.date.getUTCFullYear(), + this.date.getUTCMonth(), + this.date.getUTCDate()); + emit({date: date, session: this.session}, 1); + } + }` + mr.Reduce = reduce + err := m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, DAILY_DOWNLOADS_COLL+"_raw") + if err != nil { + return nil, err + } + var mr2 mgo.MapReduce + mr2.Map = `function() { + emit(this['_id']['date'], 1); + }` + mr2.Reduce = reduce + daily_raw := m.database.C(DAILY_DOWNLOADS_COLL + "_raw") + err = m.update(&mr2, bson.M{}, daily_raw, DAILY_DOWNLOADS_COLL) + if err != nil { + return nil, err + } + } + + var result []Visits + dailyColl := m.database.C(DAILY_DOWNLOADS_COLL) + err := dailyColl.Find(nil).All(&result) + return result, err +} + +func (m *MR) GetMonthDowloads(start time.Time, statsColl *mgo.Collection) ([]Visits, error) { + if m.isOutdated(MONTHLY_DOWNLOADS_COLL, MINUTES_UPDATE_MONTHLY) { + const reduce = `function(date, vals) { + var count = 0; + vals.forEach(function(v) { count += v; }); + return count; + }` + var mr mgo.MapReduce + mr.Map = `function() { + if (this.section == "download") { + var date = Date.UTC(this.date.getUTCFullYear(), + this.date.getUTCMonth()); + emit({date: date, session: this.session}, 1); + } + }` + mr.Reduce = reduce + err := m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, MONTHLY_DOWNLOADS_COLL+"_raw") + if err != nil { + return nil, err + } + var mr2 mgo.MapReduce + mr2.Map = `function() { + emit(this['_id']['date'], 1); + }` + mr2.Reduce = reduce + monthly_raw := m.database.C(MONTHLY_DOWNLOADS_COLL + "_raw") + err = m.update(&mr2, bson.M{}, monthly_raw, MONTHLY_DOWNLOADS_COLL) + if err != nil { + return nil, err + } + } + + var result []Visits + monthlyColl := m.database.C(MONTHLY_DOWNLOADS_COLL) + err := monthlyColl.Find(nil).All(&result) + return result, err +} + func (m *MR) update(mr *mgo.MapReduce, query bson.M, queryColl *mgo.Collection, storeColl string) error { metaColl := m.database.C(META_COLL) _, err := metaColl.RemoveAll(bson.M{"type": storeColl}) diff --git a/stats.go b/stats.go index c502de5..46ea23f 100644 --- a/stats.go +++ b/stats.go @@ -65,18 +65,24 @@ func statsHandler(h handler) { var data statsData data.S = GetStatus(h) data.S.Stats = true - data.Hourly = getHourlyVisits(h.db) - data.Daily = getDailyVisits(h.db) - data.Monthly = getMonthlyVisits(h.db) + data.HVisits = getHourlyVisits(h.db) + data.DVisits = getDailyVisits(h.db) + data.MVisits = getMonthlyVisits(h.db) + data.HDownloads = getHourlyDownloads(h.db) + data.DDownloads = getDailyDownloads(h.db) + data.MDownloads = getMonthlyDownloads(h.db) loadTemplate(h.w, "stats", data) } type statsData struct { - S Status - Hourly []visitData - Daily []visitData - Monthly []visitData + S Status + HVisits []visitData + DVisits []visitData + MVisits []visitData + HDownloads []visitData + DDownloads []visitData + MDownloads []visitData } type visitData struct { @@ -135,6 +141,57 @@ func getMonthlyVisits(db *DB) []visitData { return visits } +func getHourlyDownloads(db *DB) []visitData { + const numDays = 2 + var visits []visitData + + start := time.Now().UTC().Add(-numDays * 24 * time.Hour) + visit, _ := db.GetHourDownloads(start) + for _, v := range visit { + var elem visitData + hour := time.Unix(v.Date/1000, 0).UTC().Hour() + elem.Label = strconv.Itoa(hour + 1) + elem.Count = v.Count + visits = append(visits, elem) + } + + return visits +} + +func getDailyDownloads(db *DB) []visitData { + const numDays = 30 + var visits []visitData + + start := time.Now().UTC().Add(-numDays * 24 * time.Hour).Truncate(24 * time.Hour) + visit, _ := db.GetDayDownloads(start) + for _, v := range visit { + var elem visitData + day := time.Unix(v.Date/1000, 0).UTC().Day() + elem.Label = strconv.Itoa(day) + elem.Count = v.Count + visits = append(visits, elem) + } + + return visits +} + +func getMonthlyDownloads(db *DB) []visitData { + const numDays = 365 + var visits []visitData + + start := time.Now().UTC().Add(-numDays * 24 * time.Hour).Truncate(24 * time.Hour) + visit, _ := db.GetMonthDownloads(start) + for _, v := range visit { + var elem visitData + month := time.Unix(v.Date/1000, 0).UTC().Month() + elem.Label = month.String() + elem.Count = v.Count + visits = append(visits, elem) + } + + return visits +} + func appendFiles(r *http.Request, stats map[string]interface{}) { if r.Method == "POST" && r.MultipartForm != nil { files := r.MultipartForm.File diff --git a/templates/stats.html b/templates/stats.html index d6f8b0d..c09d4e1 100644 --- a/templates/stats.html +++ b/templates/stats.html @@ -3,12 +3,24 @@
-

Hourly visits:

- -

Daily visits:

- -

Monthly visits:

- +
+

Visits

+

Hourly:

+ +

Daily:

+ +

Monthly:

+ +
+
+

Downloads

+

Hourly:

+ +

Daily:

+ +

Monthly:

+ +
{{template "footer.html"}}