From e72de387253710fcac240a45e849ce18f124d3bf Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Sat, 30 Jul 2016 07:10:33 -0400 Subject: [PATCH] [WIP] migration to psql TODO: [ ] stats [ ] indexes --- createdb.sql | 85 +++++++++ lib/database/books.go | 346 ++++++++++++++++------------------ lib/database/books_test.go | 123 ++++++------ lib/database/database.go | 57 +++--- lib/database/database_test.go | 77 ++++++-- lib/database/mgo.go | 241 ----------------------- lib/database/news.go | 53 ++---- lib/database/news_test.go | 30 ++- lib/database/ro.go | 30 +-- lib/database/stats.go | 240 ++++------------------- lib/database/users.go | 108 ++++------- lib/database/users_test.go | 32 ++-- lib/parser/language.go | 38 ++-- lib/parser/parser.go | 49 +++-- lib/session.go | 2 +- lib/stats.go | 11 +- lib/upload.go | 24 ++- lib/user.go | 6 +- main.go | 15 +- templates/book.html | 2 +- templates/edit.html | 7 +- templates/new.html | 2 +- templates/search.html | 2 +- templates/search.opds | 4 +- 24 files changed, 648 insertions(+), 936 deletions(-) create mode 100644 createdb.sql delete mode 100644 lib/database/mgo.go diff --git a/createdb.sql b/createdb.sql new file mode 100644 index 0000000..aeb6a57 --- /dev/null +++ b/createdb.sql @@ -0,0 +1,85 @@ +CREATE TABLE books ( + id varchar(16) primary key, + title text, + authors text[], + contributor text, + publisher text, + description text, + tags text[], + date text, + lang varchar(3), + isbn varchar(13), + file_size integer, + cover boolean, + active boolean, + upload_date timestamp, + tsv tsvector +); + +-- +-- Books text search index +-- +CREATE FUNCTION books_trigger() RETURNS trigger AS $$ +declare + lang_config regconfig; +begin + lang_config := 'simple'; + if new.lang = 'da' then + lang_config := 'danish'; + elsif new.lang = 'nl' then + lang_config := 'dutch'; + elsif new.lang = 'en' then + lang_config := 'english'; + elsif new.lang = 'fi' then + lang_config := 'finnish'; + elsif new.lang = 'fr' then + lang_config := 'french'; + elsif new.lang = 'de' then + lang_config :='german'; + elsif new.lang = 'hu' then + lang_config :='hungarian'; + elsif new.lang = 'it' then + lang_config :='italian'; + elsif new.lang = 'no' then + lang_config :='norwegian'; + elsif new.lang = 'pt' then + lang_config :='portuguese'; + elsif new.lang = 'ro' then + lang_config :='romanian'; + elsif new.lang = 'ru' then + lang_config :='russian'; + elsif new.lang = 'es' then + lang_config :='spanish'; + elsif new.lang = 'sv' then + lang_config :='swedish'; + elsif new.lang = 'tr' then + lang_config :='turkish'; + end if; + + new.tsv := + setweight(to_tsvector(lang_config, coalesce(new.title,'')), 'A') || + setweight(to_tsvector('simple', coalesce(array_to_string(new.authors, ' '),'')), 'A') || + setweight(to_tsvector('simple', coalesce(new.contributor,'')), 'B') || + setweight(to_tsvector('simple', coalesce(new.publisher,'')), 'B') || + setweight(to_tsvector(lang_config, coalesce(array_to_string(new.tags, ' '),'')), 'C') || + setweight(to_tsvector(lang_config, coalesce(new.description,'')), 'D'); + return new; +end +$$ LANGUAGE plpgsql; +CREATE TRIGGER tsvectorupdate BEFORE INSERT OR UPDATE + ON books FOR EACH ROW EXECUTE PROCEDURE books_trigger(); +CREATE INDEX books_idx ON books USING GIN (tsv); + +CREATE TABLE news ( + id serial unique, + date time, + text text +); + +CREATE TABLE users ( + id serial unique, + username varchar(255) unique, + password bytea, + salt bytea, + role varchar(255) +); diff --git a/lib/database/books.go b/lib/database/books.go index 1f845a4..557df25 100644 --- a/lib/database/books.go +++ b/lib/database/books.go @@ -1,238 +1,210 @@ package database import ( - log "github.com/cihub/seelog" - "strings" "time" - - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" -) - -const ( - books_coll = "books" ) +// TODO: Author -> Authors, Subject -> Tags type Book struct { - Id string - Title string - Author []string - Contributor string - Publisher string - Description string - Subject []string - Date string - Lang []string - Isbn string - Type string - Format string - Source string - Relation string - Coverage string - Rights string - Meta string - FileSize int - Cover bool - Active bool - BadQuality int `bad_quality` - BadQualityReporters []string `bad_quality_reporters` + Id string + Title string + Author []string `sql:"authors" pg:",array"` + Contributor string + Publisher string + Description string + Subject []string `sql:"tags" pg:",array"` + Date string + Lang string + Isbn string + FileSize int + Cover bool + Active bool + UploadDate time.Time + Tsv string + //BadQuality int `bad_quality` + BadQualityReporters []string `sql:"-"` // TODO: deprecate?? } -type history struct { - Date time.Time - Changes bson.M -} +// TODO: missing history -func indexBooks(coll *mgo.Collection) { - indexes := []mgo.Index{ - { - Key: []string{"id"}, - Unique: true, - Background: true, - }, - { - Key: []string{"active", "-_id"}, - Background: true, - }, - { - Key: []string{"active", "-bad_quality", "-_id"}, - Background: true, - }, - { - Key: []string{"$text:title", "$text:author", "$text:contributor", - "$text:publisher", "$text:subject", "$text:description"}, - Weights: map[string]int{"title": 20, "author": 20, "contributor": 15, - "publisher": 15, "subject": 10, "description": 5}, - - LanguageOverride: "_lang", - Background: true, - }, - } - for _, k := range []string{"lang", "title", "author", "subject"} { - idx := mgo.Index{ - Key: []string{"active", k, "-_id"}, - Background: true, - } - indexes = append(indexes, idx) +// AddBook to the database +func (db *pgDB) AddBook(book Book) error { + emptyTime := time.Time{} + if book.UploadDate == emptyTime { + book.UploadDate = time.Now() } - for _, idx := range indexes { - err := coll.EnsureIndex(idx) - if err != nil { - log.Error("Error indexing books: ", err) - } - } + return db.sql.Create(&book) } -func addBook(coll *mgo.Collection, book map[string]interface{}) error { - book["_lang"] = metadataLang(book) - return coll.Insert(book) +// GetBooks matching query +func (db *pgDB) GetBooks(query string, length int, start int) (books []Book, num int, err error) { + return db.getBooks(true, query, length, start) } -func getBooks(coll *mgo.Collection, query string, length int, start int) (books []Book, num int, err error) { - q := buildQuery(query) - q["active"] = true - return _getBooks(coll, q, length, start) +// TODO: func (db *pgDB) GetBooksIter() Iter { + +func (db *pgDB) GetNewBooks(query string, length int, start int) (books []Book, num int, err error) { + return db.getBooks(false, query, length, start) } -func getNewBooks(coll *mgo.Collection, query string, length int, start int) (books []Book, num int, err error) { - q := buildQuery(query) - q["$nor"] = []bson.M{{"active": true}} - return _getBooks(coll, q, length, start) -} +func (db *pgDB) getBooks(active bool, query string, length int, start int) (books []Book, num int, err error) { + sqlQuery := db.sql.Model(&books) -func getBooksIter(coll *mgo.Collection) Iter { - return coll.Find(bson.M{}).Iter() -} - -func _getBooks(coll *mgo.Collection, query bson.M, length int, start int) (books []Book, num int, err error) { - q := getBookQuery(coll, query) - num, err = q.Count() - if err != nil { - return - } - if start != 0 { - q = q.Skip(start) - } - if length != 0 { - q = q.Limit(length) + searchCondition := "active = " + if active { + searchCondition = "true" + } else { + searchCondition = "false" } - err = q.All(&books) - return -} - -func getBookQuery(coll *mgo.Collection, query bson.M) *mgo.Query { - sort := []string{"$textScore:score"} - if _, present := query["bad_quality"]; present { - sort = append(sort, "-bad_quality") + params := []interface{}{} + textQuery, columnQuerys := buildQuery(query) + for _, c := range columnQuerys { + searchCondition = searchCondition + " AND " + c.column + " ILIKE ?" + params = append(params, c.value) } - sort = append(sort, "-_id") + if textQuery != "" { + searchCondition = searchCondition + " AND to_tsquery(?) @@ tsv" + params = append(params, textQuery) + } + sqlQuery = sqlQuery.Where(searchCondition, params...) - return coll.Find(query).Select(bson.M{"score": bson.M{"$meta": "textScore"}}).Sort(sort...) + if textQuery != "" { + sqlQuery = sqlQuery.Order("ts_rank(tsv, to_tsquery(?)) DESC, upload_date DESC", textQuery) + } else { + sqlQuery = sqlQuery.Order("upload_date DESC") + } + + num, err = sqlQuery. + Offset(start). + Limit(length). + SelectAndCountEstimate(100) + return books, num, err } -func getBookId(coll *mgo.Collection, id string) (Book, error) { +func (db *pgDB) GetBookId(id string) (Book, error) { var book Book - err := coll.Find(bson.M{"id": id}).One(&book) + err := db.sql.Model(&book). + Where("id = ?", id). + Select() return book, err } -func deleteBook(coll *mgo.Collection, id string) error { - return coll.Remove(bson.M{"id": id}) +func (db *pgDB) DeleteBook(id string) error { + _, err := db.sql.Model(&Book{}). + Where("id = ?", id). + Delete() + return err } -func updateBook(coll *mgo.Collection, id string, data map[string]interface{}) error { - var book map[string]interface{} - record := history{time.Now(), bson.M{}} - - err := coll.Find(bson.M{"id": id}).One(&book) - if err != nil { - return err - } - for k, _ := range data { - record.Changes[k] = book[k] - if k == "lang" { - data["_lang"] = metadataLang(data) +func (db *pgDB) UpdateBook(id string, data map[string]interface{}) error { + setCondition := "" + params := []interface{}{} + for col, val := range data { + colValid := false + for _, name := range []string{"title", "authors", "contributor", "publisher", + "description", "tags", "date", "lang", "isbn"} { + if col == name { + colValid = true + break + } } - } - - return coll.Update(bson.M{"id": id}, bson.M{"$set": data, "$push": bson.M{"history": record}}) -} - -func flagBadQuality(coll *mgo.Collection, id string, user string) error { - b, err := getBookId(coll, id) - if err != nil { - return err - } - - for _, reporter := range b.BadQualityReporters { - if reporter == user { - return nil + if !colValid { + continue } + + if len(setCondition) != 0 { + setCondition += ", " + } + setCondition += col + " = ?" + params = append(params, val) } - return coll.Update( - bson.M{"id": id}, - bson.M{ - "$inc": bson.M{"bad_quality": 1}, - "$addToSet": bson.M{"bad_quality_reporters": user}, - }, - ) + _, err := db.sql.Model(&Book{}). + Set(setCondition, params...). + Where("id = ?", id). + Update() + return err } -func activeBook(coll *mgo.Collection, id string) error { - data := map[string]interface{}{"active": true} - return coll.Update(bson.M{"id": id}, bson.M{"$set": data}) +func (db *pgDB) FlagBadQuality(id string, user string) error { + // TODO: delete me + return nil } -func isBookActive(coll *mgo.Collection, id string) bool { - var book Book - err := coll.Find(bson.M{"id": id}).One(&book) - if err != nil { +func (db *pgDB) ActiveBook(id string) error { + uploadDate := time.Now() + _, err := db.sql.Model(&Book{}). + Set("active = true, upload_date = ? ", uploadDate). + Where("id = ?", id). + Update() + return err +} + +func (db *pgDB) IsBookActive(id string) bool { + var active []bool + err := db.sql.Model(&Book{}). + Column("active"). + Where("id = ?", id). + Select(&active) + if err != nil || len(active) != 1 { return false } - return book.Active + return active[0] } -func buildQuery(q string) bson.M { - text := "" - query := bson.M{} - words := strings.Split(q, " ") +type columnq struct { + column string + value string +} + +func buildQuery(query string) (string, []columnq) { + textQuery := "" + columnQuerys := []columnq{} + words := strings.Split(query, " ") for _, w := range words { + if w == "" { + continue + } tag := strings.SplitN(w, ":", 2) - if len(tag) > 1 { - if tag[0] == "flag" { - query[tag[1]] = bson.M{"$gt": 0} - } else { - query[tag[0]] = bson.RegEx{tag[1], "i"} //FIXME: this should be a list + if len(tag) > 1 && tag[1] != "" { + value := strings.Replace(tag[1], "%", "\\%", 0) + value = strings.Replace(value, "_", "\\_", 0) + expr := "%" + value + "%" + switch tag[0] { + case "lang": + columnQuerys = append(columnQuerys, columnq{"lang", value}) + case "author": + columnQuerys = append(columnQuerys, columnq{"array_to_string(authors, ' ')", expr}) + case "title": + columnQuerys = append(columnQuerys, columnq{"title", expr}) + case "contributor": + columnQuerys = append(columnQuerys, columnq{"contributor", expr}) + case "publisher": + columnQuerys = append(columnQuerys, columnq{"publisher", expr}) + case "subject": + expr = strings.ToLower(expr) + columnQuerys = append(columnQuerys, columnq{"array_to_string(tags, ' ')", expr}) + case "tag": + expr = strings.ToLower(expr) + columnQuerys = append(columnQuerys, columnq{"array_to_string(tag, ' ')", expr}) + case "isbn": + columnQuerys = append(columnQuerys, columnq{"isbn", expr}) + case "description": + columnQuerys = append(columnQuerys, columnq{"description", expr}) } } else { - if len(text) != 0 { - text += " " + if len(textQuery) != 0 { + lastChar := textQuery[len(textQuery)-1:] + if w != "&" && w != "|" && lastChar != "&" && lastChar != "|" { + textQuery += " | " + } else { + textQuery += " " + } } - text += w + textQuery += w } } - if len(text) > 0 { - query["$text"] = bson.M{"$search": text} - } - return query -} - -func metadataLang(book map[string]interface{}) string { - text_search_langs := map[string]bool{ - "da": true, "nl": true, "en": true, "fi": true, "fr": true, "de": true, - "hu": true, "it": true, "nb": true, "pt": true, "ro": true, "ru": true, - "es": true, "sv": true, "tr": true} - - lang, ok := book["lang"].([]string) - if !ok || len(lang) == 0 || len(lang[0]) < 2 { - return "none" - } - lcode := strings.ToLower(lang[0][0:2]) - if text_search_langs[lcode] { - return lcode - } - return "none" + return textQuery, columnQuerys } diff --git a/lib/database/books_test.go b/lib/database/books_test.go index c0246a9..0c016de 100644 --- a/lib/database/books_test.go +++ b/lib/database/books_test.go @@ -2,38 +2,50 @@ package database import "testing" -var book = map[string]interface{}{ - "title": "some title", - "author": []string{"Alice", "Bob"}, - "id": "r_m-IOzzIbA6QK5w", +var book = Book{ + Id: "r_m-IOzzIbA6QK5w", + Title: "some title", + Author: []string{"Alice", "Bob"}, } -func TestAddBook(t *testing.T) { - db := Init(test_host, test_coll) - defer del(db) +func TestAddAndDeleteBook(t *testing.T) { + db, dbclose := testDbInit(t) + defer dbclose() - tAddBook(t, db) + testAddBook(t, db) books, num, err := db.GetNewBooks("", 1, 0) if err != nil { - t.Fatal("db.GetBooks() return an error: ", err) + t.Fatal("db.GetNewBooks() return an error: ", err) } if num < 1 { - t.Fatalf("db.GetBooks() didn't find any result.") + t.Fatalf("db.GetNewBooks() didn't find any result.") } if len(books) < 1 { - t.Fatalf("db.GetBooks() didn't return any result.") + t.Fatalf("db.GetNewBooks() didn't return any result.") } - if books[0].Title != book["title"] { - t.Error("Book title don't match : '", books[0].Title, "' <=> '", book["title"], "'") + if books[0].Title != book.Title { + t.Error("Book title don't match : '", books[0].Title, "' <=> '", book.Title, "'") + } + + err = db.DeleteBook(books[0].Id) + if err != nil { + t.Fatal("db.DeleteBook() return an error: ", err) + } + books, num, err = db.GetNewBooks("", 1, 0) + if err != nil { + t.Fatal("db.GetNewBooks() return an error after delete: ", err) + } + if num != 0 { + t.Fatalf("the book was not deleted.") } } func TestActiveBook(t *testing.T) { - db := Init(test_host, test_coll) - defer del(db) + db, dbclose := testDbInit(t) + defer dbclose() - tAddBook(t, db) + testAddBook(t, db) books, _, _ := db.GetNewBooks("", 1, 0) id := books[0].Id @@ -46,58 +58,57 @@ func TestActiveBook(t *testing.T) { if err != nil { t.Fatal("db.GetBookId(", id, ") return an error: ", err) } + if !b.Active { + t.Error("Book is not activated") + } if b.Author[0] != books[0].Author[0] { - t.Error("Book author don't match : '", b.Author, "' <=> '", book["author"], "'") + t.Error("Book author don't match : '", b.Author, "' <=> '", book.Author, "'") + } + + bs, num, err := db.GetBooks(book.Title, 20, 0) + if err != nil { + t.Fatal("db.GetBooks(", book.Title, ") return an error: ", err) + } + if num != 1 || len(bs) != 1 { + t.Fatal("We got a un expected number of books: ", num, bs) + } + if bs[0].Author[0] != book.Author[0] { + t.Error("Book author don't match : '", bs[0].Author, "' <=> '", book.Author, "'") + } + + bs, num, err = db.GetBooks("none", 20, 0) + if err != nil { + t.Fatal("db.GetBooks(none) return an error: ", err) + } + if num != 0 || len(bs) != 0 { + t.Error("We got books: ", num, bs) } } -func TestFlag(t *testing.T) { - db := Init(test_host, test_coll) - defer del(db) +func TestUpdateBook(t *testing.T) { + db, dbclose := testDbInit(t) + defer dbclose() - tAddBook(t, db) - id, _ := book["id"].(string) - db.ActiveBook(id) - id2 := "tfgrBvd2ps_K4iYt" - b2 := book - b2["id"] = id2 - err := db.AddBook(b2) + testAddBook(t, db) + + newTitle := "other title" + err := db.UpdateBook(book.Id, map[string]interface{}{ + "title": newTitle, + }) if err != nil { - t.Error("db.AddBook(", book, ") return an error:", err) - } - db.ActiveBook(id2) - id3 := "tfgrBvd2ps_K4iY2" - b3 := book - b3["id"] = id3 - err = db.AddBook(b3) - if err != nil { - t.Error("db.AddBook(", book, ") return an error:", err) - } - db.ActiveBook(id3) - - db.FlagBadQuality(id, "1") - db.FlagBadQuality(id, "2") - db.FlagBadQuality(id3, "1") - - b, _ := db.GetBookId(id) - if b.BadQuality != 2 { - t.Error("The bad quality flag was not increased") - } - b, _ = db.GetBookId(id3) - if b.BadQuality != 1 { - t.Error("The bad quality flag was not increased") + t.Fatal("db.UpdateBook() return an error: ", err) } - books, _, _ := db.GetBooks("flag:bad_quality", 2, 0) - if len(books) != 2 { - t.Fatal("Not the right number of results to the flag search:", len(books)) + books, num, err := db.GetNewBooks("", 1, 0) + if err != nil || num != 1 || len(books) != 1 { + t.Fatal("db.GetNewBooks() return an error: ", err) } - if books[0].Id != id { - t.Error("Search for flag bad_quality is not sort right") + if books[0].Title != newTitle { + t.Error("Book title don't match : '", books[0].Title, "' <=> '", newTitle, "'") } } -func tAddBook(t *testing.T, db DB) { +func testAddBook(t *testing.T, db DB) { err := db.AddBook(book) if err != nil { t.Error("db.AddBook(", book, ") return an error:", err) diff --git a/lib/database/database.go b/lib/database/database.go index 600fc85..a5698b2 100644 --- a/lib/database/database.go +++ b/lib/database/database.go @@ -1,19 +1,13 @@ package database import ( - log "github.com/cihub/seelog" - - "os" - - "gopkg.in/mgo.v2" + "gopkg.in/pg.v4" ) type DB interface { - Close() - Copy() DB - AddBook(book map[string]interface{}) error + Close() error + AddBook(book Book) error GetBooks(query string, length int, start int) (books []Book, num int, err error) - GetBooksIter() Iter GetNewBooks(query string, length int, start int) (books []Book, num int, err error) GetBookId(id string) (Book, error) DeleteBook(id string) error @@ -21,10 +15,12 @@ type DB interface { FlagBadQuality(id string, user string) error ActiveBook(id string) error IsBookActive(id string) bool - User(name string) *User AddUser(name string, pass string) error + GetRole(name string) (string, error) + SetPassword(name string, pass string) error + ValidPassword(name string, pass string) bool AddNews(text string) error - GetNews(num int, days int) (news []News, err error) + GetNews(num int, days int) (news []New, err error) AddStats(stats interface{}) error GetVisitedBooks() (books []Book, err error) UpdateMostVisited() error @@ -41,22 +37,33 @@ type DB interface { UpdateMonthDownloads() error } -type Iter interface { - Close() error - Next(interface{}) bool +type pgDB struct { + sql *pg.DB } -func Init(host string, name string) DB { - var err error - db := new(mgoDB) - db.session, err = mgo.Dial(host) - if err != nil { - log.Critical(err) - os.Exit(1) - } - db.name = name - db.initIndexes() - return db +// Options for the database +type Options struct { + Addr string + User string + Password string + Name string +} + +// Init the database connection +func Init(options Options) (DB, error) { + sql := pg.Connect(&pg.Options{ + Addr: options.Addr, + User: options.User, + Password: options.Password, + Database: options.Name, + }) + // TODO: create db + return &pgDB{sql}, nil +} + +// Close the database connection +func (db pgDB) Close() error { + return db.sql.Close() } func RO(db DB) DB { diff --git a/lib/database/database_test.go b/lib/database/database_test.go index cfc2a43..2b5900e 100644 --- a/lib/database/database_test.go +++ b/lib/database/database_test.go @@ -1,24 +1,71 @@ package database import ( + "io/ioutil" "testing" - - mgo "gopkg.in/mgo.v2" ) -const ( - test_coll = "test_trantor" - test_host = "127.0.0.1" -) +func testDbInit(t *testing.T) (DB, func()) { + db, err := Init(Options{ + Name: "test_trantor", + // TODO: can it be done with a local user? + User: "trantor", + Password: "trantor", + }) + if err != nil { + t.Fatal("Init() return an error: ", err) + } + pgdb, _ := db.(*pgDB) + + buf, err := ioutil.ReadFile("../../createdb.sql") + if err != nil { + t.Fatal("error reading sql schema: ", err) + } + schema := string(buf) + _, err = pgdb.sql.Exec(schema) + if err != nil { + t.Fatal("error setting up sql schema: ", err) + } + + cleanFn := func() { + entities := []struct { + name string + query string + }{ + {"table", "select tablename from pg_tables where schemaname = 'public'"}, + {"index", "select indexname from pg_indexes where schemaname = 'public'"}, + {"function", `SELECT format('%s(%s)', p.proname, pg_get_function_identity_arguments(p.oid)) + FROM pg_catalog.pg_namespace n + JOIN pg_catalog.pg_proc p + ON p.pronamespace = n.oid + WHERE n.nspname = 'public'`}, + {"trigger", "select tgname from pg_trigger"}, + } + + for _, entity := range entities { + var items []string + _, err = pgdb.sql.Query(&items, entity.query) + if err != nil { + t.Error("get the list of "+entity.name+"return an error: ", err) + } + for _, item := range items { + _, err = pgdb.sql.Exec("drop " + entity.name + " " + item + " cascade") + if err != nil { + t.Error("drop ", entity.name, " ", item, " return an error: ", err) + } + } + } + + err = db.Close() + if err != nil { + t.Error("db.Close() return an error: ", err) + } + } + + return db, cleanFn +} func TestInit(t *testing.T) { - db := Init(test_host, test_coll) - defer db.Close() -} - -func del(db DB) { - db.Close() - session, _ := mgo.Dial(test_host) - defer session.Close() - session.DB(test_coll).DropDatabase() + _, dbclose := testDbInit(t) + defer dbclose() } diff --git a/lib/database/mgo.go b/lib/database/mgo.go deleted file mode 100644 index 25fe6f1..0000000 --- a/lib/database/mgo.go +++ /dev/null @@ -1,241 +0,0 @@ -package database - -import ( - "errors" - - mgo "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" -) - -const ( - visited_coll = "visited" - downloaded_coll = "downloaded" - tags_coll = "tags" -) - -type mgoDB struct { - session *mgo.Session - name string -} - -func (db *mgoDB) initIndexes() { - dbCopy := db.session.Copy() - booksColl := dbCopy.DB(db.name).C(books_coll) - go indexBooks(booksColl) - statsColl := dbCopy.DB(db.name).C(stats_coll) - go indexStats(statsColl) - newsColl := dbCopy.DB(db.name).C(news_coll) - go indexNews(newsColl) -} - -func (db *mgoDB) Close() { - db.session.Close() -} - -func (db *mgoDB) Copy() DB { - dbCopy := new(mgoDB) - dbCopy.session = db.session.Copy() - dbCopy.name = db.name - return dbCopy -} - -func (db *mgoDB) AddBook(book map[string]interface{}) error { - booksColl := db.session.DB(db.name).C(books_coll) - return addBook(booksColl, book) -} - -func (db *mgoDB) GetBooks(query string, length int, start int) (books []Book, num int, err error) { - booksColl := db.session.DB(db.name).C(books_coll) - return getBooks(booksColl, query, length, start) -} - -func (db *mgoDB) GetBooksIter() Iter { - booksColl := db.session.DB(db.name).C(books_coll) - return getBooksIter(booksColl) -} - -func (db *mgoDB) GetNewBooks(query string, length int, start int) (books []Book, num int, err error) { - booksColl := db.session.DB(db.name).C(books_coll) - return getNewBooks(booksColl, query, length, start) -} - -func (db *mgoDB) GetBookId(id string) (Book, error) { - booksColl := db.session.DB(db.name).C(books_coll) - return getBookId(booksColl, id) -} - -func (db *mgoDB) DeleteBook(id string) error { - booksColl := db.session.DB(db.name).C(books_coll) - return deleteBook(booksColl, id) -} - -func (db *mgoDB) UpdateBook(id string, data map[string]interface{}) error { - booksColl := db.session.DB(db.name).C(books_coll) - return updateBook(booksColl, id, data) -} - -func (db *mgoDB) FlagBadQuality(id string, user string) error { - booksColl := db.session.DB(db.name).C(books_coll) - return flagBadQuality(booksColl, id, user) -} - -func (db *mgoDB) ActiveBook(id string) error { - booksColl := db.session.DB(db.name).C(books_coll) - return activeBook(booksColl, id) -} - -func (db *mgoDB) IsBookActive(id string) bool { - booksColl := db.session.DB(db.name).C(books_coll) - return isBookActive(booksColl, id) -} - -func (db *mgoDB) User(name string) *User { - userColl := db.session.DB(db.name).C(user_coll) - return getUser(userColl, name) -} - -func (db *mgoDB) AddUser(name string, pass string) error { - userColl := db.session.DB(db.name).C(user_coll) - return addUser(userColl, name, pass) -} - -func (db *mgoDB) AddNews(text string) error { - newsColl := db.session.DB(db.name).C(news_coll) - return addNews(newsColl, text) -} - -func (db *mgoDB) GetNews(num int, days int) (news []News, err error) { - newsColl := db.session.DB(db.name).C(news_coll) - return getNews(newsColl, num, days) -} - -// TODO: split code in files -func (db *mgoDB) AddStats(stats interface{}) error { - statsColl := db.session.DB(db.name).C(stats_coll) - return statsColl.Insert(stats) -} - -/* Get the most visited books - */ -func (db *mgoDB) GetVisitedBooks() (books []Book, err error) { - visitedColl := db.session.DB(db.name).C(visited_coll) - bookId, err := GetBooksVisited(visitedColl) - if err != nil { - return nil, err - } - - books = make([]Book, len(bookId)) - for i, id := range bookId { - booksColl := db.session.DB(db.name).C(books_coll) - booksColl.Find(bson.M{"_id": id}).One(&books[i]) - books[i].Id = bson.ObjectId(books[i].Id).Hex() - } - return -} - -func (db *mgoDB) UpdateMostVisited() error { - var u dbUpdate - u.src = db.session.DB(db.name).C(stats_coll) - u.dst = db.session.DB(db.name).C(visited_coll) - return u.UpdateMostBooks("book") -} - -/* Get the most downloaded books - */ -func (db *mgoDB) GetDownloadedBooks() (books []Book, err error) { - downloadedColl := db.session.DB(db.name).C(downloaded_coll) - bookId, err := GetBooksVisited(downloadedColl) - if err != nil { - return nil, err - } - - books = make([]Book, len(bookId)) - for i, id := range bookId { - booksColl := db.session.DB(db.name).C(books_coll) - booksColl.Find(bson.M{"_id": id}).One(&books[i]) - books[i].Id = bson.ObjectId(books[i].Id).Hex() - } - return -} - -func (db *mgoDB) UpdateDownloadedBooks() error { - var u dbUpdate - u.src = db.session.DB(db.name).C(stats_coll) - u.dst = db.session.DB(db.name).C(downloaded_coll) - return u.UpdateMostBooks("download") -} - -func (db *mgoDB) GetTags() ([]string, error) { - tagsColl := db.session.DB(db.name).C(tags_coll) - return GetTags(tagsColl) -} - -func (db *mgoDB) UpdateTags() error { - var u dbUpdate - u.src = db.session.DB(db.name).C(books_coll) - u.dst = db.session.DB(db.name).C(tags_coll) - return u.UpdateTags() -} - -func (db *mgoDB) GetVisits(visitType VisitType) ([]Visits, error) { - var coll *mgo.Collection - switch visitType { - case Hourly_visits: - coll = db.session.DB(db.name).C(hourly_visits_coll) - case Daily_visits: - coll = db.session.DB(db.name).C(daily_visits_coll) - case Monthly_visits: - coll = db.session.DB(db.name).C(monthly_visits_coll) - case Hourly_downloads: - coll = db.session.DB(db.name).C(hourly_downloads_coll) - case Daily_downloads: - coll = db.session.DB(db.name).C(daily_downloads_coll) - case Monthly_downloads: - coll = db.session.DB(db.name).C(monthly_downloads_coll) - default: - return nil, errors.New("Not valid VisitType") - } - return GetVisits(coll) -} - -func (db *mgoDB) UpdateHourVisits() error { - var u dbUpdate - u.src = db.session.DB(db.name).C(stats_coll) - u.dst = db.session.DB(db.name).C(hourly_visits_coll) - return u.UpdateHourVisits(false) -} - -func (db *mgoDB) UpdateDayVisits() error { - var u dbUpdate - u.src = db.session.DB(db.name).C(stats_coll) - u.dst = db.session.DB(db.name).C(daily_visits_coll) - return u.UpdateDayVisits(false) -} - -func (db *mgoDB) UpdateMonthVisits() error { - var u dbUpdate - u.src = db.session.DB(db.name).C(stats_coll) - u.dst = db.session.DB(db.name).C(monthly_visits_coll) - return u.UpdateMonthVisits(false) -} - -func (db *mgoDB) UpdateHourDownloads() error { - var u dbUpdate - u.src = db.session.DB(db.name).C(stats_coll) - u.dst = db.session.DB(db.name).C(hourly_downloads_coll) - return u.UpdateHourVisits(true) -} - -func (db *mgoDB) UpdateDayDownloads() error { - var u dbUpdate - u.src = db.session.DB(db.name).C(stats_coll) - u.dst = db.session.DB(db.name).C(daily_downloads_coll) - return u.UpdateDayVisits(true) -} - -func (db *mgoDB) UpdateMonthDownloads() error { - var u dbUpdate - u.src = db.session.DB(db.name).C(stats_coll) - u.dst = db.session.DB(db.name).C(monthly_downloads_coll) - return u.UpdateMonthVisits(true) -} diff --git a/lib/database/news.go b/lib/database/news.go index fedfabf..60617d0 100644 --- a/lib/database/news.go +++ b/lib/database/news.go @@ -1,49 +1,30 @@ package database import ( - log "github.com/cihub/seelog" - "time" - - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" ) -const ( - news_coll = "news" -) - -type News struct { +// New entry in the news table +type New struct { + ID int Date time.Time Text string } -func indexNews(coll *mgo.Collection) { - idx := mgo.Index{ - Key: []string{"-date"}, - Background: true, - } - err := coll.EnsureIndex(idx) - if err != nil { - log.Error("Error indexing news: ", err) - } +// AddNews creates a new entry +func (db *pgDB) AddNews(text string) error { + return db.sql.Create(&New{ + Text: text, + Date: time.Now(), + }) } -func addNews(coll *mgo.Collection, text string) error { - var news News - news.Text = text - news.Date = time.Now() - return coll.Insert(news) -} - -func getNews(coll *mgo.Collection, num int, days int) (news []News, err error) { - query := bson.M{} - if days != 0 { - duration := time.Duration(-24*days) * time.Hour - date := time.Now().Add(duration) - query = bson.M{"date": bson.M{"$gt": date}} - } - q := coll.Find(query).Sort("-date").Limit(num) - err = q.All(&news) - return +// GetNews returns all the news for the last 'days' limiting with a maximum of 'num' results +func (db *pgDB) GetNews(num int, days int) ([]New, error) { + var news []New + err := db.sql.Model(&news). + Limit(num). + Order("date DESC"). + Select() + return news, err } diff --git a/lib/database/news_test.go b/lib/database/news_test.go index ad74388..33cc17a 100644 --- a/lib/database/news_test.go +++ b/lib/database/news_test.go @@ -5,8 +5,8 @@ import "testing" func TestNews(t *testing.T) { const text = "Some news text" - db := Init(test_host, test_coll) - defer del(db) + db, dbclose := testDbInit(t) + defer dbclose() err := db.AddNews(text) if err != nil { @@ -24,3 +24,29 @@ func TestNews(t *testing.T) { t.Errorf("News text don't match : '", news[0].Text, "' <=> '", text, "'") } } + +func TestTwoNews(t *testing.T) { + const text = "Some news text" + const text2 = "More news" + + db, dbclose := testDbInit(t) + defer dbclose() + + err := db.AddNews(text) + if err != nil { + t.Errorf("db.News(", text, ") return an error: ", err) + } + + err = db.AddNews(text2) + if err != nil { + t.Errorf("db.News(", text, ") return an error: ", err) + } + + news, err := db.GetNews(2, 1) + if err != nil { + t.Fatalf("db.GetNews() return an error: ", err) + } + if len(news) < 2 { + t.Fatalf("No news found.") + } +} diff --git a/lib/database/ro.go b/lib/database/ro.go index c728717..9f7ac27 100644 --- a/lib/database/ro.go +++ b/lib/database/ro.go @@ -8,15 +8,11 @@ type roDB struct { db DB } -func (db *roDB) Close() { - db.db.Close() +func (db *roDB) Close() error { + return db.db.Close() } -func (db *roDB) Copy() DB { - return &roDB{db.db.Copy()} -} - -func (db *roDB) AddBook(book map[string]interface{}) error { +func (db *roDB) AddBook(book Book) error { return errors.New("RO database") } @@ -24,10 +20,6 @@ func (db *roDB) GetBooks(query string, length int, start int) (books []Book, num return db.db.GetBooks(query, length, start) } -func (db *roDB) GetBooksIter() Iter { - return db.db.GetBooksIter() -} - func (db *roDB) GetNewBooks(query string, length int, start int) (books []Book, num int, err error) { return db.db.GetNewBooks(query, length, start) } @@ -56,11 +48,19 @@ func (db *roDB) IsBookActive(id string) bool { return db.db.IsBookActive(id) } -func (db *roDB) User(name string) *User { - return db.db.User(name) +func (db *roDB) AddUser(name string, pass string) error { + return errors.New("RO database") } -func (db *roDB) AddUser(name string, pass string) error { +func (db *roDB) GetRole(name string) (string, error) { + return db.db.GetRole(name) +} + +func (db *roDB) ValidPassword(name string, pass string) bool { + return db.db.ValidPassword(name, pass) +} + +func (db *roDB) SetPassword(name string, pass string) error { return errors.New("RO database") } @@ -68,7 +68,7 @@ func (db *roDB) AddNews(text string) error { return errors.New("RO database") } -func (db *roDB) GetNews(num int, days int) (news []News, err error) { +func (db *roDB) GetNews(num int, days int) (news []New, err error) { return db.db.GetNews(num, days) } diff --git a/lib/database/stats.go b/lib/database/stats.go index a716c0b..9d866ed 100644 --- a/lib/database/stats.go +++ b/lib/database/stats.go @@ -1,33 +1,10 @@ +// TODO package database import ( - log "github.com/cihub/seelog" - "time" - - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" ) -const ( - stats_coll = "statistics" - hourly_visits_coll = "visits.hourly" - daily_visits_coll = "visits.daily" - monthly_visits_coll = "visits.monthly" - hourly_downloads_coll = "downloads.hourly" - daily_downloads_coll = "downloads.daily" - monthly_downloads_coll = "downloads.monthly" - - // FIXME: this should return to the config.go - TAGS_DISPLAY = 50 - BOOKS_FRONT_PAGE = 6 -) - -type dbUpdate struct { - src *mgo.Collection - dst *mgo.Collection -} - type VisitType int const ( @@ -44,210 +21,63 @@ type Visits struct { Count int "count" } -func indexStats(coll *mgo.Collection) { - indexes := []mgo.Index{ - { - Key: []string{"section"}, - Background: true, - }, - { - Key: []string{"-date", "section"}, - Background: true, - }, - } - - for _, idx := range indexes { - err := coll.EnsureIndex(idx) - if err != nil { - log.Error("Error indexing stats: ", err) - } - } -} - -func GetTags(tagsColl *mgo.Collection) ([]string, error) { - var result []struct { - Tag string "_id" - } - err := tagsColl.Find(nil).Sort("-count").All(&result) - if err != nil { - return nil, err - } - - tags := make([]string, len(result)) - for i, r := range result { - tags[i] = r.Tag - } - return tags, nil -} - -func GetBooksVisited(visitedColl *mgo.Collection) ([]bson.ObjectId, error) { - var result []struct { - Book bson.ObjectId "_id" - } - err := visitedColl.Find(nil).Sort("-count").All(&result) - if err != nil { - return nil, err - } - - books := make([]bson.ObjectId, len(result)) - for i, r := range result { - books[i] = r.Book - } - return books, nil -} - -func GetVisits(visitsColl *mgo.Collection) ([]Visits, error) { - var result []Visits - err := visitsColl.Find(nil).All(&result) - return result, err -} - -func (u *dbUpdate) UpdateTags() error { - var tags []struct { - Tag string "_id" - Count int "count" - } - err := u.src.Pipe([]bson.M{ - {"$project": bson.M{"subject": 1}}, - {"$unwind": "$subject"}, - {"$group": bson.M{"_id": "$subject", "count": bson.M{"$sum": 1}}}, - {"$sort": bson.M{"count": -1}}, - {"$limit": TAGS_DISPLAY}, - }).All(&tags) - if err != nil { - return err - } - - u.dst.DropCollection() - for _, tag := range tags { - err = u.dst.Insert(tag) - if err != nil { - return err - } - } +// TODO: split code in files +func (db *pgDB) AddStats(stats interface{}) error { return nil } -func (u *dbUpdate) UpdateMostBooks(section string) error { - const numDays = 30 - start := time.Now().UTC().Add(-numDays * 24 * time.Hour) +/* Get the most visited books + */ +func (db *pgDB) GetVisitedBooks() (books []Book, err error) { + return []Book{}, nil +} - var books []struct { - Book string "_id" - Count int "count" - } - err := u.src.Pipe([]bson.M{ - {"$match": bson.M{"date": bson.M{"$gt": start}, "section": section}}, - {"$project": bson.M{"id": 1}}, - {"$group": bson.M{"_id": "$id", "count": bson.M{"$sum": 1}}}, - {"$sort": bson.M{"count": -1}}, - {"$limit": BOOKS_FRONT_PAGE}, - }).All(&books) - if err != nil { - return err - } - - u.dst.DropCollection() - for _, book := range books { - err = u.dst.Insert(book) - if err != nil { - return err - } - } +func (db *pgDB) UpdateMostVisited() error { return nil } -func (u *dbUpdate) UpdateHourVisits(isDownloads bool) error { - const numDays = 2 - spanStore := numDays * 24 * time.Hour - return u.updateVisits(hourInc, spanStore, isDownloads) +/* Get the most downloaded books + */ +func (db *pgDB) GetDownloadedBooks() (books []Book, err error) { + return []Book{}, nil } -func (u *dbUpdate) UpdateDayVisits(isDownloads bool) error { - const numDays = 30 - spanStore := numDays * 24 * time.Hour - return u.updateVisits(dayInc, spanStore, isDownloads) +func (db *pgDB) UpdateDownloadedBooks() error { + return nil } -func (u *dbUpdate) UpdateMonthVisits(isDownloads bool) error { - const numDays = 365 - spanStore := numDays * 24 * time.Hour - return u.updateVisits(monthInc, spanStore, isDownloads) +func (db *pgDB) GetTags() ([]string, error) { + return []string{}, nil } -func hourInc(date time.Time) time.Time { - const span = time.Hour - return date.Add(span).Truncate(span) +func (db *pgDB) UpdateTags() error { + return nil } -func dayInc(date time.Time) time.Time { - const span = 24 * time.Hour - return date.Add(span).Truncate(span) +func (db *pgDB) GetVisits(visitType VisitType) ([]Visits, error) { + return []Visits{}, nil } -func monthInc(date time.Time) time.Time { - const span = 24 * time.Hour - return date.AddDate(0, 1, 1-date.Day()).Truncate(span) +func (db *pgDB) UpdateHourVisits() error { + return nil } -func (u *dbUpdate) updateVisits(incTime func(time.Time) time.Time, spanStore time.Duration, isDownloads bool) error { - start := u.calculateStart(spanStore) - for start.Before(time.Now().UTC()) { - stop := incTime(start) - - var count int - var err error - if isDownloads { - count, err = u.countDownloads(start, stop) - } else { - count = u.countVisits(start, stop) - } - if err != nil { - return err - } - - err = u.dst.Insert(bson.M{"date": start, "count": count}) - if err != nil { - return err - } - - start = stop - } - - _, err := u.dst.RemoveAll(bson.M{"date": bson.M{"$lt": time.Now().UTC().Add(-spanStore)}}) - return err +func (db *pgDB) UpdateDayVisits() error { + return nil } -func (u *dbUpdate) calculateStart(spanStore time.Duration) time.Time { - var date struct { - Id bson.ObjectId `bson:"_id"` - Date time.Time `bson:"date"` - } - err := u.dst.Find(bson.M{}).Sort("-date").One(&date) - if err == nil { - u.dst.RemoveId(date.Id) - return date.Date - } - return time.Now().UTC().Add(-spanStore).Truncate(time.Hour) +func (db *pgDB) UpdateMonthVisits() error { + return nil } -func (u *dbUpdate) countVisits(start time.Time, stop time.Time) int { - var result struct { - Count int "count" - } - err := u.src.Pipe([]bson.M{ - {"$match": bson.M{"date": bson.M{"$gte": start, "$lt": stop}}}, - {"$group": bson.M{"_id": "$session"}}, - {"$group": bson.M{"_id": 1, "count": bson.M{"$sum": 1}}}, - }).One(&result) - if err != nil { - return 0 - } - - return result.Count +func (db *pgDB) UpdateHourDownloads() error { + return nil } -func (u *dbUpdate) countDownloads(start time.Time, stop time.Time) (int, error) { - query := bson.M{"date": bson.M{"$gte": start, "$lt": stop}, "section": "download"} - return u.src.Find(query).Count() +func (db *pgDB) UpdateDayDownloads() error { + return nil +} + +func (db *pgDB) UpdateMonthDownloads() error { + return nil } diff --git a/lib/database/users.go b/lib/database/users.go index 88ed291..79eeab3 100644 --- a/lib/database/users.go +++ b/lib/database/users.go @@ -8,50 +8,21 @@ import ( "errors" "golang.org/x/crypto/scrypt" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" ) -const ( - user_coll = "users" - pass_salt = "ImperialLibSalt" -) - -type User struct { - user db_user - err error - coll *mgo.Collection +type user struct { + ID int + Username string + Password []byte + Salt []byte + Role string } -type db_user struct { - User string - Pass []byte - Salt []byte - Role string -} - -func getUser(coll *mgo.Collection, name string) *User { - u := new(User) - if !validUserName(name) { - u.err = errors.New("Invalid username") - return u - } - - u.coll = coll - err := u.coll.Find(bson.M{"user": name}).One(&u.user) - if err != nil { - log.Warn("Error on database checking user ", name, ": ", err) - u.err = errors.New("User not found") - return u - } - return u -} - -func addUser(coll *mgo.Collection, name string, pass string) error { +func (db *pgDB) AddUser(name string, pass string) error { if !validUserName(name) { return errors.New("Invalid user name") } - num, err := coll.Find(bson.M{"user": name}).Count() + num, err := db.sql.Model(&user{}).Where("username = ?", name).Count() if err != nil { log.Error("Error on database checking user ", name, ": ", err) return errors.New("An error happen on the database") @@ -60,41 +31,54 @@ func addUser(coll *mgo.Collection, name string, pass string) error { return errors.New("User name already exist") } - var user db_user - user.Pass, user.Salt, err = hashPass(pass) + hpass, salt, err := hashPass(pass) if err != nil { log.Error("Error hashing password: ", err) return errors.New("An error happen storing the password") } - user.User = name - user.Role = "" - return coll.Insert(user) + u := user{ + Username: name, + Password: hpass, + Salt: salt, + Role: "", + } + return db.sql.Create(&u) } -func validUserName(name string) bool { - return name != "" +func (db *pgDB) GetRole(name string) (string, error) { + var u user + err := db.sql.Model(&u).Where("username = ?", name).Select() + return u.Role, err } -func (u User) Valid(pass string) bool { - if u.err != nil { +func (db *pgDB) ValidPassword(name string, pass string) bool { + var u user + err := db.sql.Model(&u).Where("username = ?", name).Select() + if err != nil { return false } - return validatePass(pass, u.user) -} -func (u User) Role() string { - return u.user.Role -} - -func (u *User) SetPassword(pass string) error { - if u.err != nil { - return u.err + hash, err := calculateHash(pass, u.Salt) + if err != nil { + return false } + return bytes.Compare(u.Password, hash) == 0 +} + +func (db *pgDB) SetPassword(name string, pass string) error { hash, salt, err := hashPass(pass) if err != nil { return err } - return u.coll.Update(bson.M{"user": u.user.User}, bson.M{"$set": bson.M{"pass": hash, "salt": salt}}) + _, err = db.sql.Model(user{}). + Set("pass = ?, salt = ?", hash, salt). + Where("username = ?", name). + Update() + return err +} + +func validUserName(name string) bool { + return name != "" } func hashPass(pass string) (hash []byte, salt []byte, err error) { @@ -107,23 +91,13 @@ func hashPass(pass string) (hash []byte, salt []byte, err error) { } func genSalt() ([]byte, error) { - const ( - saltLen = 64 - ) + const saltLen = 64 b := make([]byte, saltLen) _, err := rand.Read(b) return b, err } -func validatePass(pass string, user db_user) bool { - hash, err := calculateHash(pass, user.Salt) - if err != nil { - return false - } - return bytes.Compare(user.Pass, hash) == 0 -} - func calculateHash(pass string, salt []byte) ([]byte, error) { const ( N = 16384 diff --git a/lib/database/users_test.go b/lib/database/users_test.go index cea542c..d3e6ab9 100644 --- a/lib/database/users_test.go +++ b/lib/database/users_test.go @@ -7,37 +7,37 @@ const ( ) func TestUserEmpty(t *testing.T) { - db := Init(test_host, test_coll) - defer del(db) + db, dbclose := testDbInit(t) + defer dbclose() - if db.User("").Valid("") { - t.Errorf("user.Valid() with an empty password return true") + if db.ValidPassword("", "") { + t.Errorf("ValidPassword() with an empty password return true") } } func TestAddUser(t *testing.T) { - db := Init(test_host, test_coll) - defer del(db) + db, dbclose := testDbInit(t) + defer dbclose() - tAddUser(t, db) - if !db.User(name).Valid(pass) { - t.Errorf("user.Valid() return false for a valid user") + testAddUser(t, db) + if !db.ValidPassword(name, pass) { + t.Errorf("ValidPassword() return false for a valid user") } } func TestEmptyUsername(t *testing.T) { - db := Init(test_host, test_coll) - defer del(db) + db, dbclose := testDbInit(t) + defer dbclose() - tAddUser(t, db) - if db.User("").Valid(pass) { - t.Errorf("user.Valid() return true for an invalid user") + testAddUser(t, db) + if db.ValidPassword("", pass) { + t.Errorf("ValidPassword() return true for an invalid user") } } -func tAddUser(t *testing.T, db DB) { +func testAddUser(t *testing.T, db DB) { err := db.AddUser(name, pass) if err != nil { - t.Errorf("db.Adduser(", name, ", ", pass, ") return an error: ", err) + t.Errorf("db.Adduser(%v, %v) return an error: %v", name, pass, err) } } diff --git a/lib/parser/language.go b/lib/parser/language.go index 23a6831..5a34785 100644 --- a/lib/parser/language.go +++ b/lib/parser/language.go @@ -3,23 +3,24 @@ package parser import ( "io/ioutil" "strings" + "unicode/utf8" "github.com/jmhodges/gocld2" "github.com/meskio/epubgo" ) -func GuessLang(epub *epubgo.Epub, orig_langs []string) []string { +func GuessLang(epub *epubgo.Epub, origLangs []string) string { spine, err := epub.Spine() if err != nil { - return orig_langs + return normalizeLangs(origLangs) } - var err_spine error - err_spine = nil + var errSpine error + errSpine = nil langs := []string{} - for err_spine == nil { + for errSpine == nil { html, err := spine.Open() - err_spine = spine.Next() + errSpine = spine.Next() if err != nil { continue } @@ -29,14 +30,16 @@ func GuessLang(epub *epubgo.Epub, orig_langs []string) []string { if err != nil { continue } - langs = append(langs, cld2.Detect(string(buff))) + if utf8.Valid(buff) { + langs = append(langs, cld2.Detect(string(buff))) + } } lang := commonLang(langs) - if lang != "un" && differentLang(lang, orig_langs) { - return []string{lang} + if lang == "un" { + return normalizeLangs(origLangs) } - return orig_langs + return lang } func commonLang(langs []string) string { @@ -56,11 +59,14 @@ func commonLang(langs []string) string { return lang } -func differentLang(lang string, orig_langs []string) bool { - orig_lang := "un" - if len(orig_langs) > 0 && len(orig_langs) >= 2 { - orig_lang = strings.ToLower(orig_langs[0][0:2]) +func normalizeLangs(langs []string) string { + lang := "un" + if len(langs) > 0 { + lang = langs[0] + if len(lang) > 3 { + lang = lang[0:2] + } + lang = strings.ToLower(lang) } - - return orig_lang != lang + return "un" } diff --git a/lib/parser/parser.go b/lib/parser/parser.go index 63556b9..75867cb 100644 --- a/lib/parser/parser.go +++ b/lib/parser/parser.go @@ -5,45 +5,46 @@ import ( "strings" "github.com/meskio/epubgo" + "gitlab.com/trantor/trantor/lib/database" ) -type MetaData map[string]interface{} - -func EpubMetadata(epub *epubgo.Epub) MetaData { - metadata := MetaData{} +func EpubMetadata(epub *epubgo.Epub) database.Book { + book := database.Book{} for _, m := range epub.MetadataFields() { data, err := epub.Metadata(m) if err != nil { continue } switch m { + case "title": + book.Title = cleanStr(strings.Join(data, ", ")) case "creator": - metadata["author"] = parseAuthr(data) + book.Author = parseAuthr(data) + case "contributor": + book.Contributor = cleanStr(strings.Join(data, ", ")) + case "publisher": + book.Publisher = cleanStr(strings.Join(data, ", ")) case "description": - metadata[m] = parseDescription(data) + book.Description = parseDescription(data) case "subject": - metadata[m] = parseSubject(data) + book.Subject = parseSubject(data) case "date": - metadata[m] = parseDate(data) + book.Date = parseDate(data) case "language": - metadata["lang"] = GuessLang(epub, data) - case "title", "contributor", "publisher": - metadata[m] = cleanStr(strings.Join(data, ", ")) + book.Lang = GuessLang(epub, data) case "identifier": attr, _ := epub.MetadataAttr(m) for i, d := range data { if attr[i]["scheme"] == "ISBN" { isbn := ISBN(d) if isbn != "" { - metadata["isbn"] = isbn + book.Isbn = isbn } } } - default: - metadata[m] = strings.Join(data, ", ") } } - return metadata + return book } func cleanStr(str string) string { @@ -88,9 +89,21 @@ func parseDescription(description []string) string { } func parseSubject(subject []string) []string { - var res []string - for _, s := range subject { - res = append(res, strings.Split(s, " / ")...) + parsed := subject + for _, sep := range []string{"/", ","} { + p2 := []string{} + for _, s := range subject { + p2 = append(p2, strings.Split(s, sep)...) + } + parsed = p2 + } + res := []string{} + for _, s := range parsed { + sub := strings.Trim(s, " ") + sub = strings.ToLower(sub) + if len(sub) != 0 { + res = append(res, sub) + } } return res } diff --git a/lib/session.go b/lib/session.go index 31cd08d..1e83167 100644 --- a/lib/session.go +++ b/lib/session.go @@ -29,7 +29,7 @@ func GetSession(r *http.Request, db database.DB) (s *Session) { s.S, err = sesStore.Get(r, "session") if err == nil && !s.S.IsNew { s.User, _ = s.S.Values["user"].(string) - s.Role = db.User(s.User).Role() + s.Role, _ = db.GetRole(s.User) } if s.S.IsNew { diff --git a/lib/stats.go b/lib/stats.go index b1b7af5..36ffd46 100644 --- a/lib/stats.go +++ b/lib/stats.go @@ -56,18 +56,16 @@ func (sg StatsGatherer) Gather(function func(handler)) func(http.ResponseWriter, return func(w http.ResponseWriter, r *http.Request) { log.Info("Query ", r.Method, " ", r.RequestURI) - db := sg.db.Copy() h := handler{ store: sg.store, - db: db, + db: sg.db, template: sg.template, hostname: sg.hostname, w: w, r: r, - sess: GetSession(r, db), + sess: GetSession(r, sg.db), ro: sg.ro, } - defer h.db.Close() function(h) sg.channel <- statsRequest{time.Now(), mux.Vars(r), h.sess, r} @@ -82,9 +80,6 @@ type statsRequest struct { } func (sg StatsGatherer) worker() { - db := sg.db.Copy() - defer db.Close() - for req := range sg.channel { stats := make(map[string]interface{}) appendFiles(req.r, stats) @@ -94,7 +89,7 @@ func (sg StatsGatherer) worker() { stats["version"] = stats_version stats["method"] = req.r.Method stats["date"] = req.date - db.AddStats(stats) + sg.db.AddStats(stats) } } diff --git a/lib/upload.go b/lib/upload.go index c9c8b84..03b8c49 100644 --- a/lib/upload.go +++ b/lib/upload.go @@ -32,11 +32,8 @@ type uploadRequest struct { } func uploadWorker(database database.DB, store storage.Store) { - db := database.Copy() - defer db.Close() - for req := range uploadChannel { - processFile(req, db, store) + processFile(req, database, store) } } @@ -50,22 +47,23 @@ func processFile(req uploadRequest, db database.DB, store storage.Store) { } defer epub.Close() - id := genId() - metadata := parser.EpubMetadata(epub) - metadata["id"] = id - metadata["cover"] = GetCover(epub, id, store) + book := parser.EpubMetadata(epub) + book.Id = genId() req.file.Seek(0, 0) - size, err := store.Store(id, req.file, epubFile) + size, err := store.Store(book.Id, req.file, epubFile) if err != nil { - log.Error("Error storing book (", id, "): ", err) + log.Error("Error storing book (", book.Id, "): ", err) return } - metadata["filesize"] = size - err = db.AddBook(metadata) + book.FileSize = int(size) + book.Cover = GetCover(epub, book.Id, store) + + err = db.AddBook(book) + log.Error(":", book.Lang, ":") if err != nil { - log.Error("Error storing metadata (", id, "): ", err) + log.Error("Error storing metadata (", book.Id, "): ", err) return } log.Info("File uploaded: ", req.filename) diff --git a/lib/user.go b/lib/user.go index 58963f4..063301b 100644 --- a/lib/user.go +++ b/lib/user.go @@ -21,7 +21,7 @@ func loginHandler(h handler) { func loginPostHandler(h handler) { user := h.r.FormValue("user") pass := h.r.FormValue("pass") - if h.db.User(user).Valid(pass) { + if h.db.ValidPassword(user, pass) { log.Info("User ", user, " log in") h.sess.LogIn(user) h.sess.Notify("Successful login!", "Welcome "+user, "success") @@ -74,12 +74,12 @@ func settingsHandler(h handler) { pass1 := h.r.FormValue("password1") pass2 := h.r.FormValue("password2") switch { - case !h.db.User(h.sess.User).Valid(current_pass): + case !h.db.ValidPassword(h.sess.User, current_pass): h.sess.Notify("Password error!", "The current password given don't match with the user password. Try again", "error") case pass1 != pass2: h.sess.Notify("Passwords don't match!", "The new password and the confirmation password don't match. Try again", "error") default: - h.db.User(h.sess.User).SetPassword(pass1) + h.db.SetPassword(h.sess.User, pass1) h.sess.Notify("Password updated!", "Your new password is correctly set.", "success") } h.sess.Save(h.w, h.r) diff --git a/main.go b/main.go index 11d3e32..927c89b 100644 --- a/main.go +++ b/main.go @@ -15,7 +15,9 @@ import ( func main() { var ( httpAddr = flag.String("addr", ":8080", "HTTP service address") - dbIP = flag.String("db-ip", "127.0.0.1", "IP address of the database") + dbAddr = flag.String("db-addr", "localhost:5432", "IP address of the database") + dbUser = flag.String("db-user", "", "User name to access the database") + dbPassword = flag.String("db-password", "", "Password to access the database") dbName = flag.String("db-name", "trantor", "Name of the database") storePath = flag.String("store", "store", "Path of the books storage") assetsPath = flag.String("assets", ".", "Path of the assets (templates, css, js, img)") @@ -32,7 +34,16 @@ func main() { } log.Info("Start the imperial library of trantor") - db := database.Init(*dbIP, *dbName) + db, err := database.Init(database.Options{ + Addr: *dbAddr, + User: *dbUser, + Password: *dbPassword, + Name: *dbName, + }) + if err != nil { + log.Critical("Problem initializing database: ", err) + os.Exit(1) + } defer db.Close() store, err := storage.Init(*storePath) diff --git a/templates/book.html b/templates/book.html index 907c8f8..3fd7b19 100644 --- a/templates/book.html +++ b/templates/book.html @@ -40,7 +40,7 @@ function delBook(){ {{if .Subject}}
Tags
{{range .Subject}}{{.}}, {{end}}
{{end}} {{if .Isbn}}
ISBN
{{.Isbn}}
{{end}} {{if .Date}}
Date
{{.Date}}
{{end}} - {{if .Lang}}
Lang
{{range .Lang}}{{.}} {{end}}
{{end}} + {{if .Lang}}
Lang
{{.Lang}}
{{end}}
diff --git a/templates/edit.html b/templates/edit.html index 6647042..776c740 100644 --- a/templates/edit.html +++ b/templates/edit.html @@ -53,12 +53,9 @@
- +
- {{range .Lang}} - - {{end}} - +
diff --git a/templates/new.html b/templates/new.html index 853df2a..04381c8 100644 --- a/templates/new.html +++ b/templates/new.html @@ -39,7 +39,7 @@ {{if .Subject}}Tags: {{range .Subject}}{{.}}, {{end}}
{{end}} {{if .Isbn}}ISBN: {{.Isbn}}
{{end}} {{if .Date}}Date: {{.Date}}
{{end}} - {{if .Lang}}Lang: {{range .Lang}}{{.}} {{end}}
{{end}} + {{if .Lang}}Lang: {{.Lang}}
{{end}} {{.Description}}

diff --git a/templates/search.html b/templates/search.html index 1d191d1..ea042dd 100644 --- a/templates/search.html +++ b/templates/search.html @@ -27,7 +27,7 @@

- {{if .Lang}}{{.Lang}}{{end}} + [{{if .Lang}}{{.Lang}}{{end}}] {{.Title}} {{if .Publisher}}{{.Publisher}}{{end}}
{{range .Author}}{{.}}, {{end}} diff --git a/templates/search.opds b/templates/search.opds index b73ce1f..6e4ff5b 100644 --- a/templates/search.opds +++ b/templates/search.opds @@ -80,8 +80,8 @@ {{.Date}} {{end}} - {{range .Lang}} - {{.}} + {{if .Lang}} + {{.Lang}} {{end}} {{range .Subject}}