[WIP] migration to psql
TODO: [ ] stats [ ] indexes
This commit is contained in:
Las Zenow
parent
e1bd235785
commit
e72de38725
24 changed files with 648 additions and 936 deletions
|
|
@ -1,238 +1,210 @@
|
|||
package database
|
||||
|
||||
import (
|
||||
log "github.com/cihub/seelog"
|
||||
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gopkg.in/mgo.v2"
|
||||
"gopkg.in/mgo.v2/bson"
|
||||
)
|
||||
|
||||
const (
|
||||
books_coll = "books"
|
||||
)
|
||||
|
||||
// TODO: Author -> Authors, Subject -> Tags
|
||||
type Book struct {
|
||||
Id string
|
||||
Title string
|
||||
Author []string
|
||||
Contributor string
|
||||
Publisher string
|
||||
Description string
|
||||
Subject []string
|
||||
Date string
|
||||
Lang []string
|
||||
Isbn string
|
||||
Type string
|
||||
Format string
|
||||
Source string
|
||||
Relation string
|
||||
Coverage string
|
||||
Rights string
|
||||
Meta string
|
||||
FileSize int
|
||||
Cover bool
|
||||
Active bool
|
||||
BadQuality int `bad_quality`
|
||||
BadQualityReporters []string `bad_quality_reporters`
|
||||
Id string
|
||||
Title string
|
||||
Author []string `sql:"authors" pg:",array"`
|
||||
Contributor string
|
||||
Publisher string
|
||||
Description string
|
||||
Subject []string `sql:"tags" pg:",array"`
|
||||
Date string
|
||||
Lang string
|
||||
Isbn string
|
||||
FileSize int
|
||||
Cover bool
|
||||
Active bool
|
||||
UploadDate time.Time
|
||||
Tsv string
|
||||
//BadQuality int `bad_quality`
|
||||
BadQualityReporters []string `sql:"-"` // TODO: deprecate??
|
||||
}
|
||||
|
||||
type history struct {
|
||||
Date time.Time
|
||||
Changes bson.M
|
||||
}
|
||||
// TODO: missing history
|
||||
|
||||
func indexBooks(coll *mgo.Collection) {
|
||||
indexes := []mgo.Index{
|
||||
{
|
||||
Key: []string{"id"},
|
||||
Unique: true,
|
||||
Background: true,
|
||||
},
|
||||
{
|
||||
Key: []string{"active", "-_id"},
|
||||
Background: true,
|
||||
},
|
||||
{
|
||||
Key: []string{"active", "-bad_quality", "-_id"},
|
||||
Background: true,
|
||||
},
|
||||
{
|
||||
Key: []string{"$text:title", "$text:author", "$text:contributor",
|
||||
"$text:publisher", "$text:subject", "$text:description"},
|
||||
Weights: map[string]int{"title": 20, "author": 20, "contributor": 15,
|
||||
"publisher": 15, "subject": 10, "description": 5},
|
||||
|
||||
LanguageOverride: "_lang",
|
||||
Background: true,
|
||||
},
|
||||
}
|
||||
for _, k := range []string{"lang", "title", "author", "subject"} {
|
||||
idx := mgo.Index{
|
||||
Key: []string{"active", k, "-_id"},
|
||||
Background: true,
|
||||
}
|
||||
indexes = append(indexes, idx)
|
||||
// AddBook to the database
|
||||
func (db *pgDB) AddBook(book Book) error {
|
||||
emptyTime := time.Time{}
|
||||
if book.UploadDate == emptyTime {
|
||||
book.UploadDate = time.Now()
|
||||
}
|
||||
|
||||
for _, idx := range indexes {
|
||||
err := coll.EnsureIndex(idx)
|
||||
if err != nil {
|
||||
log.Error("Error indexing books: ", err)
|
||||
}
|
||||
}
|
||||
return db.sql.Create(&book)
|
||||
}
|
||||
|
||||
func addBook(coll *mgo.Collection, book map[string]interface{}) error {
|
||||
book["_lang"] = metadataLang(book)
|
||||
return coll.Insert(book)
|
||||
// GetBooks matching query
|
||||
func (db *pgDB) GetBooks(query string, length int, start int) (books []Book, num int, err error) {
|
||||
return db.getBooks(true, query, length, start)
|
||||
}
|
||||
|
||||
func getBooks(coll *mgo.Collection, query string, length int, start int) (books []Book, num int, err error) {
|
||||
q := buildQuery(query)
|
||||
q["active"] = true
|
||||
return _getBooks(coll, q, length, start)
|
||||
// TODO: func (db *pgDB) GetBooksIter() Iter {
|
||||
|
||||
func (db *pgDB) GetNewBooks(query string, length int, start int) (books []Book, num int, err error) {
|
||||
return db.getBooks(false, query, length, start)
|
||||
}
|
||||
|
||||
func getNewBooks(coll *mgo.Collection, query string, length int, start int) (books []Book, num int, err error) {
|
||||
q := buildQuery(query)
|
||||
q["$nor"] = []bson.M{{"active": true}}
|
||||
return _getBooks(coll, q, length, start)
|
||||
}
|
||||
func (db *pgDB) getBooks(active bool, query string, length int, start int) (books []Book, num int, err error) {
|
||||
sqlQuery := db.sql.Model(&books)
|
||||
|
||||
func getBooksIter(coll *mgo.Collection) Iter {
|
||||
return coll.Find(bson.M{}).Iter()
|
||||
}
|
||||
|
||||
func _getBooks(coll *mgo.Collection, query bson.M, length int, start int) (books []Book, num int, err error) {
|
||||
q := getBookQuery(coll, query)
|
||||
num, err = q.Count()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if start != 0 {
|
||||
q = q.Skip(start)
|
||||
}
|
||||
if length != 0 {
|
||||
q = q.Limit(length)
|
||||
searchCondition := "active = "
|
||||
if active {
|
||||
searchCondition = "true"
|
||||
} else {
|
||||
searchCondition = "false"
|
||||
}
|
||||
|
||||
err = q.All(&books)
|
||||
return
|
||||
}
|
||||
|
||||
func getBookQuery(coll *mgo.Collection, query bson.M) *mgo.Query {
|
||||
sort := []string{"$textScore:score"}
|
||||
if _, present := query["bad_quality"]; present {
|
||||
sort = append(sort, "-bad_quality")
|
||||
params := []interface{}{}
|
||||
textQuery, columnQuerys := buildQuery(query)
|
||||
for _, c := range columnQuerys {
|
||||
searchCondition = searchCondition + " AND " + c.column + " ILIKE ?"
|
||||
params = append(params, c.value)
|
||||
}
|
||||
sort = append(sort, "-_id")
|
||||
if textQuery != "" {
|
||||
searchCondition = searchCondition + " AND to_tsquery(?) @@ tsv"
|
||||
params = append(params, textQuery)
|
||||
}
|
||||
sqlQuery = sqlQuery.Where(searchCondition, params...)
|
||||
|
||||
return coll.Find(query).Select(bson.M{"score": bson.M{"$meta": "textScore"}}).Sort(sort...)
|
||||
if textQuery != "" {
|
||||
sqlQuery = sqlQuery.Order("ts_rank(tsv, to_tsquery(?)) DESC, upload_date DESC", textQuery)
|
||||
} else {
|
||||
sqlQuery = sqlQuery.Order("upload_date DESC")
|
||||
}
|
||||
|
||||
num, err = sqlQuery.
|
||||
Offset(start).
|
||||
Limit(length).
|
||||
SelectAndCountEstimate(100)
|
||||
return books, num, err
|
||||
}
|
||||
|
||||
func getBookId(coll *mgo.Collection, id string) (Book, error) {
|
||||
func (db *pgDB) GetBookId(id string) (Book, error) {
|
||||
var book Book
|
||||
err := coll.Find(bson.M{"id": id}).One(&book)
|
||||
err := db.sql.Model(&book).
|
||||
Where("id = ?", id).
|
||||
Select()
|
||||
return book, err
|
||||
}
|
||||
|
||||
func deleteBook(coll *mgo.Collection, id string) error {
|
||||
return coll.Remove(bson.M{"id": id})
|
||||
func (db *pgDB) DeleteBook(id string) error {
|
||||
_, err := db.sql.Model(&Book{}).
|
||||
Where("id = ?", id).
|
||||
Delete()
|
||||
return err
|
||||
}
|
||||
|
||||
func updateBook(coll *mgo.Collection, id string, data map[string]interface{}) error {
|
||||
var book map[string]interface{}
|
||||
record := history{time.Now(), bson.M{}}
|
||||
|
||||
err := coll.Find(bson.M{"id": id}).One(&book)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for k, _ := range data {
|
||||
record.Changes[k] = book[k]
|
||||
if k == "lang" {
|
||||
data["_lang"] = metadataLang(data)
|
||||
func (db *pgDB) UpdateBook(id string, data map[string]interface{}) error {
|
||||
setCondition := ""
|
||||
params := []interface{}{}
|
||||
for col, val := range data {
|
||||
colValid := false
|
||||
for _, name := range []string{"title", "authors", "contributor", "publisher",
|
||||
"description", "tags", "date", "lang", "isbn"} {
|
||||
if col == name {
|
||||
colValid = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return coll.Update(bson.M{"id": id}, bson.M{"$set": data, "$push": bson.M{"history": record}})
|
||||
}
|
||||
|
||||
func flagBadQuality(coll *mgo.Collection, id string, user string) error {
|
||||
b, err := getBookId(coll, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, reporter := range b.BadQualityReporters {
|
||||
if reporter == user {
|
||||
return nil
|
||||
if !colValid {
|
||||
continue
|
||||
}
|
||||
|
||||
if len(setCondition) != 0 {
|
||||
setCondition += ", "
|
||||
}
|
||||
setCondition += col + " = ?"
|
||||
params = append(params, val)
|
||||
}
|
||||
return coll.Update(
|
||||
bson.M{"id": id},
|
||||
bson.M{
|
||||
"$inc": bson.M{"bad_quality": 1},
|
||||
"$addToSet": bson.M{"bad_quality_reporters": user},
|
||||
},
|
||||
)
|
||||
_, err := db.sql.Model(&Book{}).
|
||||
Set(setCondition, params...).
|
||||
Where("id = ?", id).
|
||||
Update()
|
||||
return err
|
||||
}
|
||||
|
||||
func activeBook(coll *mgo.Collection, id string) error {
|
||||
data := map[string]interface{}{"active": true}
|
||||
return coll.Update(bson.M{"id": id}, bson.M{"$set": data})
|
||||
func (db *pgDB) FlagBadQuality(id string, user string) error {
|
||||
// TODO: delete me
|
||||
return nil
|
||||
}
|
||||
|
||||
func isBookActive(coll *mgo.Collection, id string) bool {
|
||||
var book Book
|
||||
err := coll.Find(bson.M{"id": id}).One(&book)
|
||||
if err != nil {
|
||||
func (db *pgDB) ActiveBook(id string) error {
|
||||
uploadDate := time.Now()
|
||||
_, err := db.sql.Model(&Book{}).
|
||||
Set("active = true, upload_date = ? ", uploadDate).
|
||||
Where("id = ?", id).
|
||||
Update()
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *pgDB) IsBookActive(id string) bool {
|
||||
var active []bool
|
||||
err := db.sql.Model(&Book{}).
|
||||
Column("active").
|
||||
Where("id = ?", id).
|
||||
Select(&active)
|
||||
if err != nil || len(active) != 1 {
|
||||
return false
|
||||
}
|
||||
return book.Active
|
||||
return active[0]
|
||||
}
|
||||
|
||||
func buildQuery(q string) bson.M {
|
||||
text := ""
|
||||
query := bson.M{}
|
||||
words := strings.Split(q, " ")
|
||||
type columnq struct {
|
||||
column string
|
||||
value string
|
||||
}
|
||||
|
||||
func buildQuery(query string) (string, []columnq) {
|
||||
textQuery := ""
|
||||
columnQuerys := []columnq{}
|
||||
words := strings.Split(query, " ")
|
||||
for _, w := range words {
|
||||
if w == "" {
|
||||
continue
|
||||
}
|
||||
tag := strings.SplitN(w, ":", 2)
|
||||
if len(tag) > 1 {
|
||||
if tag[0] == "flag" {
|
||||
query[tag[1]] = bson.M{"$gt": 0}
|
||||
} else {
|
||||
query[tag[0]] = bson.RegEx{tag[1], "i"} //FIXME: this should be a list
|
||||
if len(tag) > 1 && tag[1] != "" {
|
||||
value := strings.Replace(tag[1], "%", "\\%", 0)
|
||||
value = strings.Replace(value, "_", "\\_", 0)
|
||||
expr := "%" + value + "%"
|
||||
switch tag[0] {
|
||||
case "lang":
|
||||
columnQuerys = append(columnQuerys, columnq{"lang", value})
|
||||
case "author":
|
||||
columnQuerys = append(columnQuerys, columnq{"array_to_string(authors, ' ')", expr})
|
||||
case "title":
|
||||
columnQuerys = append(columnQuerys, columnq{"title", expr})
|
||||
case "contributor":
|
||||
columnQuerys = append(columnQuerys, columnq{"contributor", expr})
|
||||
case "publisher":
|
||||
columnQuerys = append(columnQuerys, columnq{"publisher", expr})
|
||||
case "subject":
|
||||
expr = strings.ToLower(expr)
|
||||
columnQuerys = append(columnQuerys, columnq{"array_to_string(tags, ' ')", expr})
|
||||
case "tag":
|
||||
expr = strings.ToLower(expr)
|
||||
columnQuerys = append(columnQuerys, columnq{"array_to_string(tag, ' ')", expr})
|
||||
case "isbn":
|
||||
columnQuerys = append(columnQuerys, columnq{"isbn", expr})
|
||||
case "description":
|
||||
columnQuerys = append(columnQuerys, columnq{"description", expr})
|
||||
}
|
||||
} else {
|
||||
if len(text) != 0 {
|
||||
text += " "
|
||||
if len(textQuery) != 0 {
|
||||
lastChar := textQuery[len(textQuery)-1:]
|
||||
if w != "&" && w != "|" && lastChar != "&" && lastChar != "|" {
|
||||
textQuery += " | "
|
||||
} else {
|
||||
textQuery += " "
|
||||
}
|
||||
}
|
||||
text += w
|
||||
textQuery += w
|
||||
}
|
||||
}
|
||||
if len(text) > 0 {
|
||||
query["$text"] = bson.M{"$search": text}
|
||||
}
|
||||
return query
|
||||
}
|
||||
|
||||
func metadataLang(book map[string]interface{}) string {
|
||||
text_search_langs := map[string]bool{
|
||||
"da": true, "nl": true, "en": true, "fi": true, "fr": true, "de": true,
|
||||
"hu": true, "it": true, "nb": true, "pt": true, "ro": true, "ru": true,
|
||||
"es": true, "sv": true, "tr": true}
|
||||
|
||||
lang, ok := book["lang"].([]string)
|
||||
if !ok || len(lang) == 0 || len(lang[0]) < 2 {
|
||||
return "none"
|
||||
}
|
||||
lcode := strings.ToLower(lang[0][0:2])
|
||||
if text_search_langs[lcode] {
|
||||
return lcode
|
||||
}
|
||||
return "none"
|
||||
return textQuery, columnQuerys
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,38 +2,50 @@ package database
|
|||
|
||||
import "testing"
|
||||
|
||||
var book = map[string]interface{}{
|
||||
"title": "some title",
|
||||
"author": []string{"Alice", "Bob"},
|
||||
"id": "r_m-IOzzIbA6QK5w",
|
||||
var book = Book{
|
||||
Id: "r_m-IOzzIbA6QK5w",
|
||||
Title: "some title",
|
||||
Author: []string{"Alice", "Bob"},
|
||||
}
|
||||
|
||||
func TestAddBook(t *testing.T) {
|
||||
db := Init(test_host, test_coll)
|
||||
defer del(db)
|
||||
func TestAddAndDeleteBook(t *testing.T) {
|
||||
db, dbclose := testDbInit(t)
|
||||
defer dbclose()
|
||||
|
||||
tAddBook(t, db)
|
||||
testAddBook(t, db)
|
||||
|
||||
books, num, err := db.GetNewBooks("", 1, 0)
|
||||
if err != nil {
|
||||
t.Fatal("db.GetBooks() return an error: ", err)
|
||||
t.Fatal("db.GetNewBooks() return an error: ", err)
|
||||
}
|
||||
if num < 1 {
|
||||
t.Fatalf("db.GetBooks() didn't find any result.")
|
||||
t.Fatalf("db.GetNewBooks() didn't find any result.")
|
||||
}
|
||||
if len(books) < 1 {
|
||||
t.Fatalf("db.GetBooks() didn't return any result.")
|
||||
t.Fatalf("db.GetNewBooks() didn't return any result.")
|
||||
}
|
||||
if books[0].Title != book["title"] {
|
||||
t.Error("Book title don't match : '", books[0].Title, "' <=> '", book["title"], "'")
|
||||
if books[0].Title != book.Title {
|
||||
t.Error("Book title don't match : '", books[0].Title, "' <=> '", book.Title, "'")
|
||||
}
|
||||
|
||||
err = db.DeleteBook(books[0].Id)
|
||||
if err != nil {
|
||||
t.Fatal("db.DeleteBook() return an error: ", err)
|
||||
}
|
||||
books, num, err = db.GetNewBooks("", 1, 0)
|
||||
if err != nil {
|
||||
t.Fatal("db.GetNewBooks() return an error after delete: ", err)
|
||||
}
|
||||
if num != 0 {
|
||||
t.Fatalf("the book was not deleted.")
|
||||
}
|
||||
}
|
||||
|
||||
func TestActiveBook(t *testing.T) {
|
||||
db := Init(test_host, test_coll)
|
||||
defer del(db)
|
||||
db, dbclose := testDbInit(t)
|
||||
defer dbclose()
|
||||
|
||||
tAddBook(t, db)
|
||||
testAddBook(t, db)
|
||||
books, _, _ := db.GetNewBooks("", 1, 0)
|
||||
id := books[0].Id
|
||||
|
||||
|
|
@ -46,58 +58,57 @@ func TestActiveBook(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatal("db.GetBookId(", id, ") return an error: ", err)
|
||||
}
|
||||
if !b.Active {
|
||||
t.Error("Book is not activated")
|
||||
}
|
||||
if b.Author[0] != books[0].Author[0] {
|
||||
t.Error("Book author don't match : '", b.Author, "' <=> '", book["author"], "'")
|
||||
t.Error("Book author don't match : '", b.Author, "' <=> '", book.Author, "'")
|
||||
}
|
||||
|
||||
bs, num, err := db.GetBooks(book.Title, 20, 0)
|
||||
if err != nil {
|
||||
t.Fatal("db.GetBooks(", book.Title, ") return an error: ", err)
|
||||
}
|
||||
if num != 1 || len(bs) != 1 {
|
||||
t.Fatal("We got a un expected number of books: ", num, bs)
|
||||
}
|
||||
if bs[0].Author[0] != book.Author[0] {
|
||||
t.Error("Book author don't match : '", bs[0].Author, "' <=> '", book.Author, "'")
|
||||
}
|
||||
|
||||
bs, num, err = db.GetBooks("none", 20, 0)
|
||||
if err != nil {
|
||||
t.Fatal("db.GetBooks(none) return an error: ", err)
|
||||
}
|
||||
if num != 0 || len(bs) != 0 {
|
||||
t.Error("We got books: ", num, bs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlag(t *testing.T) {
|
||||
db := Init(test_host, test_coll)
|
||||
defer del(db)
|
||||
func TestUpdateBook(t *testing.T) {
|
||||
db, dbclose := testDbInit(t)
|
||||
defer dbclose()
|
||||
|
||||
tAddBook(t, db)
|
||||
id, _ := book["id"].(string)
|
||||
db.ActiveBook(id)
|
||||
id2 := "tfgrBvd2ps_K4iYt"
|
||||
b2 := book
|
||||
b2["id"] = id2
|
||||
err := db.AddBook(b2)
|
||||
testAddBook(t, db)
|
||||
|
||||
newTitle := "other title"
|
||||
err := db.UpdateBook(book.Id, map[string]interface{}{
|
||||
"title": newTitle,
|
||||
})
|
||||
if err != nil {
|
||||
t.Error("db.AddBook(", book, ") return an error:", err)
|
||||
}
|
||||
db.ActiveBook(id2)
|
||||
id3 := "tfgrBvd2ps_K4iY2"
|
||||
b3 := book
|
||||
b3["id"] = id3
|
||||
err = db.AddBook(b3)
|
||||
if err != nil {
|
||||
t.Error("db.AddBook(", book, ") return an error:", err)
|
||||
}
|
||||
db.ActiveBook(id3)
|
||||
|
||||
db.FlagBadQuality(id, "1")
|
||||
db.FlagBadQuality(id, "2")
|
||||
db.FlagBadQuality(id3, "1")
|
||||
|
||||
b, _ := db.GetBookId(id)
|
||||
if b.BadQuality != 2 {
|
||||
t.Error("The bad quality flag was not increased")
|
||||
}
|
||||
b, _ = db.GetBookId(id3)
|
||||
if b.BadQuality != 1 {
|
||||
t.Error("The bad quality flag was not increased")
|
||||
t.Fatal("db.UpdateBook() return an error: ", err)
|
||||
}
|
||||
|
||||
books, _, _ := db.GetBooks("flag:bad_quality", 2, 0)
|
||||
if len(books) != 2 {
|
||||
t.Fatal("Not the right number of results to the flag search:", len(books))
|
||||
books, num, err := db.GetNewBooks("", 1, 0)
|
||||
if err != nil || num != 1 || len(books) != 1 {
|
||||
t.Fatal("db.GetNewBooks() return an error: ", err)
|
||||
}
|
||||
if books[0].Id != id {
|
||||
t.Error("Search for flag bad_quality is not sort right")
|
||||
if books[0].Title != newTitle {
|
||||
t.Error("Book title don't match : '", books[0].Title, "' <=> '", newTitle, "'")
|
||||
}
|
||||
}
|
||||
|
||||
func tAddBook(t *testing.T, db DB) {
|
||||
func testAddBook(t *testing.T, db DB) {
|
||||
err := db.AddBook(book)
|
||||
if err != nil {
|
||||
t.Error("db.AddBook(", book, ") return an error:", err)
|
||||
|
|
|
|||
|
|
@ -1,19 +1,13 @@
|
|||
package database
|
||||
|
||||
import (
|
||||
log "github.com/cihub/seelog"
|
||||
|
||||
"os"
|
||||
|
||||
"gopkg.in/mgo.v2"
|
||||
"gopkg.in/pg.v4"
|
||||
)
|
||||
|
||||
type DB interface {
|
||||
Close()
|
||||
Copy() DB
|
||||
AddBook(book map[string]interface{}) error
|
||||
Close() error
|
||||
AddBook(book Book) error
|
||||
GetBooks(query string, length int, start int) (books []Book, num int, err error)
|
||||
GetBooksIter() Iter
|
||||
GetNewBooks(query string, length int, start int) (books []Book, num int, err error)
|
||||
GetBookId(id string) (Book, error)
|
||||
DeleteBook(id string) error
|
||||
|
|
@ -21,10 +15,12 @@ type DB interface {
|
|||
FlagBadQuality(id string, user string) error
|
||||
ActiveBook(id string) error
|
||||
IsBookActive(id string) bool
|
||||
User(name string) *User
|
||||
AddUser(name string, pass string) error
|
||||
GetRole(name string) (string, error)
|
||||
SetPassword(name string, pass string) error
|
||||
ValidPassword(name string, pass string) bool
|
||||
AddNews(text string) error
|
||||
GetNews(num int, days int) (news []News, err error)
|
||||
GetNews(num int, days int) (news []New, err error)
|
||||
AddStats(stats interface{}) error
|
||||
GetVisitedBooks() (books []Book, err error)
|
||||
UpdateMostVisited() error
|
||||
|
|
@ -41,22 +37,33 @@ type DB interface {
|
|||
UpdateMonthDownloads() error
|
||||
}
|
||||
|
||||
type Iter interface {
|
||||
Close() error
|
||||
Next(interface{}) bool
|
||||
type pgDB struct {
|
||||
sql *pg.DB
|
||||
}
|
||||
|
||||
func Init(host string, name string) DB {
|
||||
var err error
|
||||
db := new(mgoDB)
|
||||
db.session, err = mgo.Dial(host)
|
||||
if err != nil {
|
||||
log.Critical(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
db.name = name
|
||||
db.initIndexes()
|
||||
return db
|
||||
// Options for the database
|
||||
type Options struct {
|
||||
Addr string
|
||||
User string
|
||||
Password string
|
||||
Name string
|
||||
}
|
||||
|
||||
// Init the database connection
|
||||
func Init(options Options) (DB, error) {
|
||||
sql := pg.Connect(&pg.Options{
|
||||
Addr: options.Addr,
|
||||
User: options.User,
|
||||
Password: options.Password,
|
||||
Database: options.Name,
|
||||
})
|
||||
// TODO: create db
|
||||
return &pgDB{sql}, nil
|
||||
}
|
||||
|
||||
// Close the database connection
|
||||
func (db pgDB) Close() error {
|
||||
return db.sql.Close()
|
||||
}
|
||||
|
||||
func RO(db DB) DB {
|
||||
|
|
|
|||
|
|
@ -1,24 +1,71 @@
|
|||
package database
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"testing"
|
||||
|
||||
mgo "gopkg.in/mgo.v2"
|
||||
)
|
||||
|
||||
const (
|
||||
test_coll = "test_trantor"
|
||||
test_host = "127.0.0.1"
|
||||
)
|
||||
func testDbInit(t *testing.T) (DB, func()) {
|
||||
db, err := Init(Options{
|
||||
Name: "test_trantor",
|
||||
// TODO: can it be done with a local user?
|
||||
User: "trantor",
|
||||
Password: "trantor",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal("Init() return an error: ", err)
|
||||
}
|
||||
pgdb, _ := db.(*pgDB)
|
||||
|
||||
buf, err := ioutil.ReadFile("../../createdb.sql")
|
||||
if err != nil {
|
||||
t.Fatal("error reading sql schema: ", err)
|
||||
}
|
||||
schema := string(buf)
|
||||
_, err = pgdb.sql.Exec(schema)
|
||||
if err != nil {
|
||||
t.Fatal("error setting up sql schema: ", err)
|
||||
}
|
||||
|
||||
cleanFn := func() {
|
||||
entities := []struct {
|
||||
name string
|
||||
query string
|
||||
}{
|
||||
{"table", "select tablename from pg_tables where schemaname = 'public'"},
|
||||
{"index", "select indexname from pg_indexes where schemaname = 'public'"},
|
||||
{"function", `SELECT format('%s(%s)', p.proname, pg_get_function_identity_arguments(p.oid))
|
||||
FROM pg_catalog.pg_namespace n
|
||||
JOIN pg_catalog.pg_proc p
|
||||
ON p.pronamespace = n.oid
|
||||
WHERE n.nspname = 'public'`},
|
||||
{"trigger", "select tgname from pg_trigger"},
|
||||
}
|
||||
|
||||
for _, entity := range entities {
|
||||
var items []string
|
||||
_, err = pgdb.sql.Query(&items, entity.query)
|
||||
if err != nil {
|
||||
t.Error("get the list of "+entity.name+"return an error: ", err)
|
||||
}
|
||||
for _, item := range items {
|
||||
_, err = pgdb.sql.Exec("drop " + entity.name + " " + item + " cascade")
|
||||
if err != nil {
|
||||
t.Error("drop ", entity.name, " ", item, " return an error: ", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
err = db.Close()
|
||||
if err != nil {
|
||||
t.Error("db.Close() return an error: ", err)
|
||||
}
|
||||
}
|
||||
|
||||
return db, cleanFn
|
||||
}
|
||||
|
||||
func TestInit(t *testing.T) {
|
||||
db := Init(test_host, test_coll)
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
func del(db DB) {
|
||||
db.Close()
|
||||
session, _ := mgo.Dial(test_host)
|
||||
defer session.Close()
|
||||
session.DB(test_coll).DropDatabase()
|
||||
_, dbclose := testDbInit(t)
|
||||
defer dbclose()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,241 +0,0 @@
|
|||
package database
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
mgo "gopkg.in/mgo.v2"
|
||||
"gopkg.in/mgo.v2/bson"
|
||||
)
|
||||
|
||||
const (
|
||||
visited_coll = "visited"
|
||||
downloaded_coll = "downloaded"
|
||||
tags_coll = "tags"
|
||||
)
|
||||
|
||||
type mgoDB struct {
|
||||
session *mgo.Session
|
||||
name string
|
||||
}
|
||||
|
||||
func (db *mgoDB) initIndexes() {
|
||||
dbCopy := db.session.Copy()
|
||||
booksColl := dbCopy.DB(db.name).C(books_coll)
|
||||
go indexBooks(booksColl)
|
||||
statsColl := dbCopy.DB(db.name).C(stats_coll)
|
||||
go indexStats(statsColl)
|
||||
newsColl := dbCopy.DB(db.name).C(news_coll)
|
||||
go indexNews(newsColl)
|
||||
}
|
||||
|
||||
func (db *mgoDB) Close() {
|
||||
db.session.Close()
|
||||
}
|
||||
|
||||
func (db *mgoDB) Copy() DB {
|
||||
dbCopy := new(mgoDB)
|
||||
dbCopy.session = db.session.Copy()
|
||||
dbCopy.name = db.name
|
||||
return dbCopy
|
||||
}
|
||||
|
||||
func (db *mgoDB) AddBook(book map[string]interface{}) error {
|
||||
booksColl := db.session.DB(db.name).C(books_coll)
|
||||
return addBook(booksColl, book)
|
||||
}
|
||||
|
||||
func (db *mgoDB) GetBooks(query string, length int, start int) (books []Book, num int, err error) {
|
||||
booksColl := db.session.DB(db.name).C(books_coll)
|
||||
return getBooks(booksColl, query, length, start)
|
||||
}
|
||||
|
||||
func (db *mgoDB) GetBooksIter() Iter {
|
||||
booksColl := db.session.DB(db.name).C(books_coll)
|
||||
return getBooksIter(booksColl)
|
||||
}
|
||||
|
||||
func (db *mgoDB) GetNewBooks(query string, length int, start int) (books []Book, num int, err error) {
|
||||
booksColl := db.session.DB(db.name).C(books_coll)
|
||||
return getNewBooks(booksColl, query, length, start)
|
||||
}
|
||||
|
||||
func (db *mgoDB) GetBookId(id string) (Book, error) {
|
||||
booksColl := db.session.DB(db.name).C(books_coll)
|
||||
return getBookId(booksColl, id)
|
||||
}
|
||||
|
||||
func (db *mgoDB) DeleteBook(id string) error {
|
||||
booksColl := db.session.DB(db.name).C(books_coll)
|
||||
return deleteBook(booksColl, id)
|
||||
}
|
||||
|
||||
func (db *mgoDB) UpdateBook(id string, data map[string]interface{}) error {
|
||||
booksColl := db.session.DB(db.name).C(books_coll)
|
||||
return updateBook(booksColl, id, data)
|
||||
}
|
||||
|
||||
func (db *mgoDB) FlagBadQuality(id string, user string) error {
|
||||
booksColl := db.session.DB(db.name).C(books_coll)
|
||||
return flagBadQuality(booksColl, id, user)
|
||||
}
|
||||
|
||||
func (db *mgoDB) ActiveBook(id string) error {
|
||||
booksColl := db.session.DB(db.name).C(books_coll)
|
||||
return activeBook(booksColl, id)
|
||||
}
|
||||
|
||||
func (db *mgoDB) IsBookActive(id string) bool {
|
||||
booksColl := db.session.DB(db.name).C(books_coll)
|
||||
return isBookActive(booksColl, id)
|
||||
}
|
||||
|
||||
func (db *mgoDB) User(name string) *User {
|
||||
userColl := db.session.DB(db.name).C(user_coll)
|
||||
return getUser(userColl, name)
|
||||
}
|
||||
|
||||
func (db *mgoDB) AddUser(name string, pass string) error {
|
||||
userColl := db.session.DB(db.name).C(user_coll)
|
||||
return addUser(userColl, name, pass)
|
||||
}
|
||||
|
||||
func (db *mgoDB) AddNews(text string) error {
|
||||
newsColl := db.session.DB(db.name).C(news_coll)
|
||||
return addNews(newsColl, text)
|
||||
}
|
||||
|
||||
func (db *mgoDB) GetNews(num int, days int) (news []News, err error) {
|
||||
newsColl := db.session.DB(db.name).C(news_coll)
|
||||
return getNews(newsColl, num, days)
|
||||
}
|
||||
|
||||
// TODO: split code in files
|
||||
func (db *mgoDB) AddStats(stats interface{}) error {
|
||||
statsColl := db.session.DB(db.name).C(stats_coll)
|
||||
return statsColl.Insert(stats)
|
||||
}
|
||||
|
||||
/* Get the most visited books
|
||||
*/
|
||||
func (db *mgoDB) GetVisitedBooks() (books []Book, err error) {
|
||||
visitedColl := db.session.DB(db.name).C(visited_coll)
|
||||
bookId, err := GetBooksVisited(visitedColl)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
books = make([]Book, len(bookId))
|
||||
for i, id := range bookId {
|
||||
booksColl := db.session.DB(db.name).C(books_coll)
|
||||
booksColl.Find(bson.M{"_id": id}).One(&books[i])
|
||||
books[i].Id = bson.ObjectId(books[i].Id).Hex()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (db *mgoDB) UpdateMostVisited() error {
|
||||
var u dbUpdate
|
||||
u.src = db.session.DB(db.name).C(stats_coll)
|
||||
u.dst = db.session.DB(db.name).C(visited_coll)
|
||||
return u.UpdateMostBooks("book")
|
||||
}
|
||||
|
||||
/* Get the most downloaded books
|
||||
*/
|
||||
func (db *mgoDB) GetDownloadedBooks() (books []Book, err error) {
|
||||
downloadedColl := db.session.DB(db.name).C(downloaded_coll)
|
||||
bookId, err := GetBooksVisited(downloadedColl)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
books = make([]Book, len(bookId))
|
||||
for i, id := range bookId {
|
||||
booksColl := db.session.DB(db.name).C(books_coll)
|
||||
booksColl.Find(bson.M{"_id": id}).One(&books[i])
|
||||
books[i].Id = bson.ObjectId(books[i].Id).Hex()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (db *mgoDB) UpdateDownloadedBooks() error {
|
||||
var u dbUpdate
|
||||
u.src = db.session.DB(db.name).C(stats_coll)
|
||||
u.dst = db.session.DB(db.name).C(downloaded_coll)
|
||||
return u.UpdateMostBooks("download")
|
||||
}
|
||||
|
||||
func (db *mgoDB) GetTags() ([]string, error) {
|
||||
tagsColl := db.session.DB(db.name).C(tags_coll)
|
||||
return GetTags(tagsColl)
|
||||
}
|
||||
|
||||
func (db *mgoDB) UpdateTags() error {
|
||||
var u dbUpdate
|
||||
u.src = db.session.DB(db.name).C(books_coll)
|
||||
u.dst = db.session.DB(db.name).C(tags_coll)
|
||||
return u.UpdateTags()
|
||||
}
|
||||
|
||||
func (db *mgoDB) GetVisits(visitType VisitType) ([]Visits, error) {
|
||||
var coll *mgo.Collection
|
||||
switch visitType {
|
||||
case Hourly_visits:
|
||||
coll = db.session.DB(db.name).C(hourly_visits_coll)
|
||||
case Daily_visits:
|
||||
coll = db.session.DB(db.name).C(daily_visits_coll)
|
||||
case Monthly_visits:
|
||||
coll = db.session.DB(db.name).C(monthly_visits_coll)
|
||||
case Hourly_downloads:
|
||||
coll = db.session.DB(db.name).C(hourly_downloads_coll)
|
||||
case Daily_downloads:
|
||||
coll = db.session.DB(db.name).C(daily_downloads_coll)
|
||||
case Monthly_downloads:
|
||||
coll = db.session.DB(db.name).C(monthly_downloads_coll)
|
||||
default:
|
||||
return nil, errors.New("Not valid VisitType")
|
||||
}
|
||||
return GetVisits(coll)
|
||||
}
|
||||
|
||||
func (db *mgoDB) UpdateHourVisits() error {
|
||||
var u dbUpdate
|
||||
u.src = db.session.DB(db.name).C(stats_coll)
|
||||
u.dst = db.session.DB(db.name).C(hourly_visits_coll)
|
||||
return u.UpdateHourVisits(false)
|
||||
}
|
||||
|
||||
func (db *mgoDB) UpdateDayVisits() error {
|
||||
var u dbUpdate
|
||||
u.src = db.session.DB(db.name).C(stats_coll)
|
||||
u.dst = db.session.DB(db.name).C(daily_visits_coll)
|
||||
return u.UpdateDayVisits(false)
|
||||
}
|
||||
|
||||
func (db *mgoDB) UpdateMonthVisits() error {
|
||||
var u dbUpdate
|
||||
u.src = db.session.DB(db.name).C(stats_coll)
|
||||
u.dst = db.session.DB(db.name).C(monthly_visits_coll)
|
||||
return u.UpdateMonthVisits(false)
|
||||
}
|
||||
|
||||
func (db *mgoDB) UpdateHourDownloads() error {
|
||||
var u dbUpdate
|
||||
u.src = db.session.DB(db.name).C(stats_coll)
|
||||
u.dst = db.session.DB(db.name).C(hourly_downloads_coll)
|
||||
return u.UpdateHourVisits(true)
|
||||
}
|
||||
|
||||
func (db *mgoDB) UpdateDayDownloads() error {
|
||||
var u dbUpdate
|
||||
u.src = db.session.DB(db.name).C(stats_coll)
|
||||
u.dst = db.session.DB(db.name).C(daily_downloads_coll)
|
||||
return u.UpdateDayVisits(true)
|
||||
}
|
||||
|
||||
func (db *mgoDB) UpdateMonthDownloads() error {
|
||||
var u dbUpdate
|
||||
u.src = db.session.DB(db.name).C(stats_coll)
|
||||
u.dst = db.session.DB(db.name).C(monthly_downloads_coll)
|
||||
return u.UpdateMonthVisits(true)
|
||||
}
|
||||
|
|
@ -1,49 +1,30 @@
|
|||
package database
|
||||
|
||||
import (
|
||||
log "github.com/cihub/seelog"
|
||||
|
||||
"time"
|
||||
|
||||
"gopkg.in/mgo.v2"
|
||||
"gopkg.in/mgo.v2/bson"
|
||||
)
|
||||
|
||||
const (
|
||||
news_coll = "news"
|
||||
)
|
||||
|
||||
type News struct {
|
||||
// New entry in the news table
|
||||
type New struct {
|
||||
ID int
|
||||
Date time.Time
|
||||
Text string
|
||||
}
|
||||
|
||||
func indexNews(coll *mgo.Collection) {
|
||||
idx := mgo.Index{
|
||||
Key: []string{"-date"},
|
||||
Background: true,
|
||||
}
|
||||
err := coll.EnsureIndex(idx)
|
||||
if err != nil {
|
||||
log.Error("Error indexing news: ", err)
|
||||
}
|
||||
// AddNews creates a new entry
|
||||
func (db *pgDB) AddNews(text string) error {
|
||||
return db.sql.Create(&New{
|
||||
Text: text,
|
||||
Date: time.Now(),
|
||||
})
|
||||
}
|
||||
|
||||
func addNews(coll *mgo.Collection, text string) error {
|
||||
var news News
|
||||
news.Text = text
|
||||
news.Date = time.Now()
|
||||
return coll.Insert(news)
|
||||
}
|
||||
|
||||
func getNews(coll *mgo.Collection, num int, days int) (news []News, err error) {
|
||||
query := bson.M{}
|
||||
if days != 0 {
|
||||
duration := time.Duration(-24*days) * time.Hour
|
||||
date := time.Now().Add(duration)
|
||||
query = bson.M{"date": bson.M{"$gt": date}}
|
||||
}
|
||||
q := coll.Find(query).Sort("-date").Limit(num)
|
||||
err = q.All(&news)
|
||||
return
|
||||
// GetNews returns all the news for the last 'days' limiting with a maximum of 'num' results
|
||||
func (db *pgDB) GetNews(num int, days int) ([]New, error) {
|
||||
var news []New
|
||||
err := db.sql.Model(&news).
|
||||
Limit(num).
|
||||
Order("date DESC").
|
||||
Select()
|
||||
return news, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,8 +5,8 @@ import "testing"
|
|||
func TestNews(t *testing.T) {
|
||||
const text = "Some news text"
|
||||
|
||||
db := Init(test_host, test_coll)
|
||||
defer del(db)
|
||||
db, dbclose := testDbInit(t)
|
||||
defer dbclose()
|
||||
|
||||
err := db.AddNews(text)
|
||||
if err != nil {
|
||||
|
|
@ -24,3 +24,29 @@ func TestNews(t *testing.T) {
|
|||
t.Errorf("News text don't match : '", news[0].Text, "' <=> '", text, "'")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoNews(t *testing.T) {
|
||||
const text = "Some news text"
|
||||
const text2 = "More news"
|
||||
|
||||
db, dbclose := testDbInit(t)
|
||||
defer dbclose()
|
||||
|
||||
err := db.AddNews(text)
|
||||
if err != nil {
|
||||
t.Errorf("db.News(", text, ") return an error: ", err)
|
||||
}
|
||||
|
||||
err = db.AddNews(text2)
|
||||
if err != nil {
|
||||
t.Errorf("db.News(", text, ") return an error: ", err)
|
||||
}
|
||||
|
||||
news, err := db.GetNews(2, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("db.GetNews() return an error: ", err)
|
||||
}
|
||||
if len(news) < 2 {
|
||||
t.Fatalf("No news found.")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,15 +8,11 @@ type roDB struct {
|
|||
db DB
|
||||
}
|
||||
|
||||
func (db *roDB) Close() {
|
||||
db.db.Close()
|
||||
func (db *roDB) Close() error {
|
||||
return db.db.Close()
|
||||
}
|
||||
|
||||
func (db *roDB) Copy() DB {
|
||||
return &roDB{db.db.Copy()}
|
||||
}
|
||||
|
||||
func (db *roDB) AddBook(book map[string]interface{}) error {
|
||||
func (db *roDB) AddBook(book Book) error {
|
||||
return errors.New("RO database")
|
||||
}
|
||||
|
||||
|
|
@ -24,10 +20,6 @@ func (db *roDB) GetBooks(query string, length int, start int) (books []Book, num
|
|||
return db.db.GetBooks(query, length, start)
|
||||
}
|
||||
|
||||
func (db *roDB) GetBooksIter() Iter {
|
||||
return db.db.GetBooksIter()
|
||||
}
|
||||
|
||||
func (db *roDB) GetNewBooks(query string, length int, start int) (books []Book, num int, err error) {
|
||||
return db.db.GetNewBooks(query, length, start)
|
||||
}
|
||||
|
|
@ -56,11 +48,19 @@ func (db *roDB) IsBookActive(id string) bool {
|
|||
return db.db.IsBookActive(id)
|
||||
}
|
||||
|
||||
func (db *roDB) User(name string) *User {
|
||||
return db.db.User(name)
|
||||
func (db *roDB) AddUser(name string, pass string) error {
|
||||
return errors.New("RO database")
|
||||
}
|
||||
|
||||
func (db *roDB) AddUser(name string, pass string) error {
|
||||
func (db *roDB) GetRole(name string) (string, error) {
|
||||
return db.db.GetRole(name)
|
||||
}
|
||||
|
||||
func (db *roDB) ValidPassword(name string, pass string) bool {
|
||||
return db.db.ValidPassword(name, pass)
|
||||
}
|
||||
|
||||
func (db *roDB) SetPassword(name string, pass string) error {
|
||||
return errors.New("RO database")
|
||||
}
|
||||
|
||||
|
|
@ -68,7 +68,7 @@ func (db *roDB) AddNews(text string) error {
|
|||
return errors.New("RO database")
|
||||
}
|
||||
|
||||
func (db *roDB) GetNews(num int, days int) (news []News, err error) {
|
||||
func (db *roDB) GetNews(num int, days int) (news []New, err error) {
|
||||
return db.db.GetNews(num, days)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,33 +1,10 @@
|
|||
// TODO
|
||||
package database
|
||||
|
||||
import (
|
||||
log "github.com/cihub/seelog"
|
||||
|
||||
"time"
|
||||
|
||||
"gopkg.in/mgo.v2"
|
||||
"gopkg.in/mgo.v2/bson"
|
||||
)
|
||||
|
||||
const (
|
||||
stats_coll = "statistics"
|
||||
hourly_visits_coll = "visits.hourly"
|
||||
daily_visits_coll = "visits.daily"
|
||||
monthly_visits_coll = "visits.monthly"
|
||||
hourly_downloads_coll = "downloads.hourly"
|
||||
daily_downloads_coll = "downloads.daily"
|
||||
monthly_downloads_coll = "downloads.monthly"
|
||||
|
||||
// FIXME: this should return to the config.go
|
||||
TAGS_DISPLAY = 50
|
||||
BOOKS_FRONT_PAGE = 6
|
||||
)
|
||||
|
||||
type dbUpdate struct {
|
||||
src *mgo.Collection
|
||||
dst *mgo.Collection
|
||||
}
|
||||
|
||||
type VisitType int
|
||||
|
||||
const (
|
||||
|
|
@ -44,210 +21,63 @@ type Visits struct {
|
|||
Count int "count"
|
||||
}
|
||||
|
||||
func indexStats(coll *mgo.Collection) {
|
||||
indexes := []mgo.Index{
|
||||
{
|
||||
Key: []string{"section"},
|
||||
Background: true,
|
||||
},
|
||||
{
|
||||
Key: []string{"-date", "section"},
|
||||
Background: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, idx := range indexes {
|
||||
err := coll.EnsureIndex(idx)
|
||||
if err != nil {
|
||||
log.Error("Error indexing stats: ", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func GetTags(tagsColl *mgo.Collection) ([]string, error) {
|
||||
var result []struct {
|
||||
Tag string "_id"
|
||||
}
|
||||
err := tagsColl.Find(nil).Sort("-count").All(&result)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tags := make([]string, len(result))
|
||||
for i, r := range result {
|
||||
tags[i] = r.Tag
|
||||
}
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
func GetBooksVisited(visitedColl *mgo.Collection) ([]bson.ObjectId, error) {
|
||||
var result []struct {
|
||||
Book bson.ObjectId "_id"
|
||||
}
|
||||
err := visitedColl.Find(nil).Sort("-count").All(&result)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
books := make([]bson.ObjectId, len(result))
|
||||
for i, r := range result {
|
||||
books[i] = r.Book
|
||||
}
|
||||
return books, nil
|
||||
}
|
||||
|
||||
func GetVisits(visitsColl *mgo.Collection) ([]Visits, error) {
|
||||
var result []Visits
|
||||
err := visitsColl.Find(nil).All(&result)
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (u *dbUpdate) UpdateTags() error {
|
||||
var tags []struct {
|
||||
Tag string "_id"
|
||||
Count int "count"
|
||||
}
|
||||
err := u.src.Pipe([]bson.M{
|
||||
{"$project": bson.M{"subject": 1}},
|
||||
{"$unwind": "$subject"},
|
||||
{"$group": bson.M{"_id": "$subject", "count": bson.M{"$sum": 1}}},
|
||||
{"$sort": bson.M{"count": -1}},
|
||||
{"$limit": TAGS_DISPLAY},
|
||||
}).All(&tags)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
u.dst.DropCollection()
|
||||
for _, tag := range tags {
|
||||
err = u.dst.Insert(tag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// TODO: split code in files
|
||||
func (db *pgDB) AddStats(stats interface{}) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *dbUpdate) UpdateMostBooks(section string) error {
|
||||
const numDays = 30
|
||||
start := time.Now().UTC().Add(-numDays * 24 * time.Hour)
|
||||
/* Get the most visited books
|
||||
*/
|
||||
func (db *pgDB) GetVisitedBooks() (books []Book, err error) {
|
||||
return []Book{}, nil
|
||||
}
|
||||
|
||||
var books []struct {
|
||||
Book string "_id"
|
||||
Count int "count"
|
||||
}
|
||||
err := u.src.Pipe([]bson.M{
|
||||
{"$match": bson.M{"date": bson.M{"$gt": start}, "section": section}},
|
||||
{"$project": bson.M{"id": 1}},
|
||||
{"$group": bson.M{"_id": "$id", "count": bson.M{"$sum": 1}}},
|
||||
{"$sort": bson.M{"count": -1}},
|
||||
{"$limit": BOOKS_FRONT_PAGE},
|
||||
}).All(&books)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
u.dst.DropCollection()
|
||||
for _, book := range books {
|
||||
err = u.dst.Insert(book)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
func (db *pgDB) UpdateMostVisited() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *dbUpdate) UpdateHourVisits(isDownloads bool) error {
|
||||
const numDays = 2
|
||||
spanStore := numDays * 24 * time.Hour
|
||||
return u.updateVisits(hourInc, spanStore, isDownloads)
|
||||
/* Get the most downloaded books
|
||||
*/
|
||||
func (db *pgDB) GetDownloadedBooks() (books []Book, err error) {
|
||||
return []Book{}, nil
|
||||
}
|
||||
|
||||
func (u *dbUpdate) UpdateDayVisits(isDownloads bool) error {
|
||||
const numDays = 30
|
||||
spanStore := numDays * 24 * time.Hour
|
||||
return u.updateVisits(dayInc, spanStore, isDownloads)
|
||||
func (db *pgDB) UpdateDownloadedBooks() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *dbUpdate) UpdateMonthVisits(isDownloads bool) error {
|
||||
const numDays = 365
|
||||
spanStore := numDays * 24 * time.Hour
|
||||
return u.updateVisits(monthInc, spanStore, isDownloads)
|
||||
func (db *pgDB) GetTags() ([]string, error) {
|
||||
return []string{}, nil
|
||||
}
|
||||
|
||||
func hourInc(date time.Time) time.Time {
|
||||
const span = time.Hour
|
||||
return date.Add(span).Truncate(span)
|
||||
func (db *pgDB) UpdateTags() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func dayInc(date time.Time) time.Time {
|
||||
const span = 24 * time.Hour
|
||||
return date.Add(span).Truncate(span)
|
||||
func (db *pgDB) GetVisits(visitType VisitType) ([]Visits, error) {
|
||||
return []Visits{}, nil
|
||||
}
|
||||
|
||||
func monthInc(date time.Time) time.Time {
|
||||
const span = 24 * time.Hour
|
||||
return date.AddDate(0, 1, 1-date.Day()).Truncate(span)
|
||||
func (db *pgDB) UpdateHourVisits() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *dbUpdate) updateVisits(incTime func(time.Time) time.Time, spanStore time.Duration, isDownloads bool) error {
|
||||
start := u.calculateStart(spanStore)
|
||||
for start.Before(time.Now().UTC()) {
|
||||
stop := incTime(start)
|
||||
|
||||
var count int
|
||||
var err error
|
||||
if isDownloads {
|
||||
count, err = u.countDownloads(start, stop)
|
||||
} else {
|
||||
count = u.countVisits(start, stop)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = u.dst.Insert(bson.M{"date": start, "count": count})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
start = stop
|
||||
}
|
||||
|
||||
_, err := u.dst.RemoveAll(bson.M{"date": bson.M{"$lt": time.Now().UTC().Add(-spanStore)}})
|
||||
return err
|
||||
func (db *pgDB) UpdateDayVisits() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *dbUpdate) calculateStart(spanStore time.Duration) time.Time {
|
||||
var date struct {
|
||||
Id bson.ObjectId `bson:"_id"`
|
||||
Date time.Time `bson:"date"`
|
||||
}
|
||||
err := u.dst.Find(bson.M{}).Sort("-date").One(&date)
|
||||
if err == nil {
|
||||
u.dst.RemoveId(date.Id)
|
||||
return date.Date
|
||||
}
|
||||
return time.Now().UTC().Add(-spanStore).Truncate(time.Hour)
|
||||
func (db *pgDB) UpdateMonthVisits() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *dbUpdate) countVisits(start time.Time, stop time.Time) int {
|
||||
var result struct {
|
||||
Count int "count"
|
||||
}
|
||||
err := u.src.Pipe([]bson.M{
|
||||
{"$match": bson.M{"date": bson.M{"$gte": start, "$lt": stop}}},
|
||||
{"$group": bson.M{"_id": "$session"}},
|
||||
{"$group": bson.M{"_id": 1, "count": bson.M{"$sum": 1}}},
|
||||
}).One(&result)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
return result.Count
|
||||
func (db *pgDB) UpdateHourDownloads() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *dbUpdate) countDownloads(start time.Time, stop time.Time) (int, error) {
|
||||
query := bson.M{"date": bson.M{"$gte": start, "$lt": stop}, "section": "download"}
|
||||
return u.src.Find(query).Count()
|
||||
func (db *pgDB) UpdateDayDownloads() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *pgDB) UpdateMonthDownloads() error {
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,50 +8,21 @@ import (
|
|||
"errors"
|
||||
|
||||
"golang.org/x/crypto/scrypt"
|
||||
"gopkg.in/mgo.v2"
|
||||
"gopkg.in/mgo.v2/bson"
|
||||
)
|
||||
|
||||
const (
|
||||
user_coll = "users"
|
||||
pass_salt = "ImperialLibSalt"
|
||||
)
|
||||
|
||||
type User struct {
|
||||
user db_user
|
||||
err error
|
||||
coll *mgo.Collection
|
||||
type user struct {
|
||||
ID int
|
||||
Username string
|
||||
Password []byte
|
||||
Salt []byte
|
||||
Role string
|
||||
}
|
||||
|
||||
type db_user struct {
|
||||
User string
|
||||
Pass []byte
|
||||
Salt []byte
|
||||
Role string
|
||||
}
|
||||
|
||||
func getUser(coll *mgo.Collection, name string) *User {
|
||||
u := new(User)
|
||||
if !validUserName(name) {
|
||||
u.err = errors.New("Invalid username")
|
||||
return u
|
||||
}
|
||||
|
||||
u.coll = coll
|
||||
err := u.coll.Find(bson.M{"user": name}).One(&u.user)
|
||||
if err != nil {
|
||||
log.Warn("Error on database checking user ", name, ": ", err)
|
||||
u.err = errors.New("User not found")
|
||||
return u
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
func addUser(coll *mgo.Collection, name string, pass string) error {
|
||||
func (db *pgDB) AddUser(name string, pass string) error {
|
||||
if !validUserName(name) {
|
||||
return errors.New("Invalid user name")
|
||||
}
|
||||
num, err := coll.Find(bson.M{"user": name}).Count()
|
||||
num, err := db.sql.Model(&user{}).Where("username = ?", name).Count()
|
||||
if err != nil {
|
||||
log.Error("Error on database checking user ", name, ": ", err)
|
||||
return errors.New("An error happen on the database")
|
||||
|
|
@ -60,41 +31,54 @@ func addUser(coll *mgo.Collection, name string, pass string) error {
|
|||
return errors.New("User name already exist")
|
||||
}
|
||||
|
||||
var user db_user
|
||||
user.Pass, user.Salt, err = hashPass(pass)
|
||||
hpass, salt, err := hashPass(pass)
|
||||
if err != nil {
|
||||
log.Error("Error hashing password: ", err)
|
||||
return errors.New("An error happen storing the password")
|
||||
}
|
||||
user.User = name
|
||||
user.Role = ""
|
||||
return coll.Insert(user)
|
||||
u := user{
|
||||
Username: name,
|
||||
Password: hpass,
|
||||
Salt: salt,
|
||||
Role: "",
|
||||
}
|
||||
return db.sql.Create(&u)
|
||||
}
|
||||
|
||||
func validUserName(name string) bool {
|
||||
return name != ""
|
||||
func (db *pgDB) GetRole(name string) (string, error) {
|
||||
var u user
|
||||
err := db.sql.Model(&u).Where("username = ?", name).Select()
|
||||
return u.Role, err
|
||||
}
|
||||
|
||||
func (u User) Valid(pass string) bool {
|
||||
if u.err != nil {
|
||||
func (db *pgDB) ValidPassword(name string, pass string) bool {
|
||||
var u user
|
||||
err := db.sql.Model(&u).Where("username = ?", name).Select()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return validatePass(pass, u.user)
|
||||
}
|
||||
|
||||
func (u User) Role() string {
|
||||
return u.user.Role
|
||||
}
|
||||
|
||||
func (u *User) SetPassword(pass string) error {
|
||||
if u.err != nil {
|
||||
return u.err
|
||||
hash, err := calculateHash(pass, u.Salt)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return bytes.Compare(u.Password, hash) == 0
|
||||
}
|
||||
|
||||
func (db *pgDB) SetPassword(name string, pass string) error {
|
||||
hash, salt, err := hashPass(pass)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return u.coll.Update(bson.M{"user": u.user.User}, bson.M{"$set": bson.M{"pass": hash, "salt": salt}})
|
||||
_, err = db.sql.Model(user{}).
|
||||
Set("pass = ?, salt = ?", hash, salt).
|
||||
Where("username = ?", name).
|
||||
Update()
|
||||
return err
|
||||
}
|
||||
|
||||
func validUserName(name string) bool {
|
||||
return name != ""
|
||||
}
|
||||
|
||||
func hashPass(pass string) (hash []byte, salt []byte, err error) {
|
||||
|
|
@ -107,23 +91,13 @@ func hashPass(pass string) (hash []byte, salt []byte, err error) {
|
|||
}
|
||||
|
||||
func genSalt() ([]byte, error) {
|
||||
const (
|
||||
saltLen = 64
|
||||
)
|
||||
const saltLen = 64
|
||||
|
||||
b := make([]byte, saltLen)
|
||||
_, err := rand.Read(b)
|
||||
return b, err
|
||||
}
|
||||
|
||||
func validatePass(pass string, user db_user) bool {
|
||||
hash, err := calculateHash(pass, user.Salt)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return bytes.Compare(user.Pass, hash) == 0
|
||||
}
|
||||
|
||||
func calculateHash(pass string, salt []byte) ([]byte, error) {
|
||||
const (
|
||||
N = 16384
|
||||
|
|
|
|||
|
|
@ -7,37 +7,37 @@ const (
|
|||
)
|
||||
|
||||
func TestUserEmpty(t *testing.T) {
|
||||
db := Init(test_host, test_coll)
|
||||
defer del(db)
|
||||
db, dbclose := testDbInit(t)
|
||||
defer dbclose()
|
||||
|
||||
if db.User("").Valid("") {
|
||||
t.Errorf("user.Valid() with an empty password return true")
|
||||
if db.ValidPassword("", "") {
|
||||
t.Errorf("ValidPassword() with an empty password return true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddUser(t *testing.T) {
|
||||
db := Init(test_host, test_coll)
|
||||
defer del(db)
|
||||
db, dbclose := testDbInit(t)
|
||||
defer dbclose()
|
||||
|
||||
tAddUser(t, db)
|
||||
if !db.User(name).Valid(pass) {
|
||||
t.Errorf("user.Valid() return false for a valid user")
|
||||
testAddUser(t, db)
|
||||
if !db.ValidPassword(name, pass) {
|
||||
t.Errorf("ValidPassword() return false for a valid user")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyUsername(t *testing.T) {
|
||||
db := Init(test_host, test_coll)
|
||||
defer del(db)
|
||||
db, dbclose := testDbInit(t)
|
||||
defer dbclose()
|
||||
|
||||
tAddUser(t, db)
|
||||
if db.User("").Valid(pass) {
|
||||
t.Errorf("user.Valid() return true for an invalid user")
|
||||
testAddUser(t, db)
|
||||
if db.ValidPassword("", pass) {
|
||||
t.Errorf("ValidPassword() return true for an invalid user")
|
||||
}
|
||||
}
|
||||
|
||||
func tAddUser(t *testing.T, db DB) {
|
||||
func testAddUser(t *testing.T, db DB) {
|
||||
err := db.AddUser(name, pass)
|
||||
if err != nil {
|
||||
t.Errorf("db.Adduser(", name, ", ", pass, ") return an error: ", err)
|
||||
t.Errorf("db.Adduser(%v, %v) return an error: %v", name, pass, err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Reference in a new issue