From f4690b2bbad6f3cbe2a196bd9954e531ae8c55cc Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Thu, 21 Aug 2014 19:24:23 -0500 Subject: [PATCH] Move files to a local folder --- .gitignore | 1 + admin.go | 14 +++-- config.go | 7 ++- cover.go | 79 ++++++++++++--------------- database/books.go | 69 +++++++++++++++--------- database/books_test.go | 55 +++++++++++++++++-- database/database.go | 13 ++--- reader.go | 29 ++++++++-- stats.go | 58 ++++++++++++-------- storage/dir.go | 42 +++++++++++++++ storage/storage.go | 57 ++++++++++++++++++++ storage/storage_test.go | 113 +++++++++++++++++++++++++++++++++++++++ store.go | 115 ---------------------------------------- templates/index.html | 6 +-- templates/new.html | 2 +- templates/search.html | 2 +- trantor.go | 81 +++++++++++++++------------- upload.go | 101 ++++++++++++++++++++++++++++------- 18 files changed, 556 insertions(+), 288 deletions(-) create mode 100644 storage/dir.go create mode 100644 storage/storage.go create mode 100644 storage/storage_test.go delete mode 100644 store.go diff --git a/.gitignore b/.gitignore index 76fdaad..eaa3020 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ trantor.git +store/ tools/adduser/adduser tools/update/update tools/togridfs/togridfs diff --git a/admin.go b/admin.go index 6015dfe..e93a010 100644 --- a/admin.go +++ b/admin.go @@ -25,7 +25,7 @@ func deleteHandler(h handler) { h.sess.Notify("Book not found!", "The book with id '"+id+"' is not there", "error") continue } - DeleteBook(book, h.db) + h.store.Delete(id) h.db.DeleteBook(id) if !book.Active { @@ -101,7 +101,7 @@ func saveHandler(h handler) { h.sess.Notify("Book Modified!", "", "success") h.sess.Save(h.w, h.r) - if h.db.BookActive(id) { + if h.db.IsBookActive(id) { http.Redirect(h.w, h.r, "/book/"+id, http.StatusFound) } else { http.Redirect(h.w, h.r, "/new/", http.StatusFound) @@ -174,17 +174,25 @@ func storeHandler(h handler) { var titles []string ids := strings.Split(mux.Vars(h.r)["ids"], "/") for _, id := range ids { + if id == "" { + continue + } book, err := h.db.GetBookId(id) if err != nil { h.sess.Notify("Book not found!", "The book with id '"+id+"' is not there", "error") continue } + if err != nil { + h.sess.Notify("An error ocurred!", err.Error(), "error") + log.Error("Error getting book for storing '", book.Title, "': ", err.Error()) + continue + } + err = h.db.ActiveBook(id) if err != nil { h.sess.Notify("An error ocurred!", err.Error(), "error") log.Error("Error storing book '", book.Title, "': ", err.Error()) continue } - h.db.UpdateBook(id, map[string]interface{}{"active": true}) titles = append(titles, book.Title) } if titles != nil { diff --git a/config.go b/config.go index bd27d3a..81b7487 100644 --- a/config.go +++ b/config.go @@ -6,8 +6,10 @@ const ( DB_IP = "127.0.0.1" DB_NAME = "trantor" META_COLL = "meta" - FS_BOOKS = "fs_books" - FS_IMGS = "fs_imgs" + + EPUB_FILE = "book.epub" + COVER_FILE = "cover.jpg" + COVER_SMALL_FILE = "coverSmall.jpg" MINUTES_UPDATE_TAGS = 11 MINUTES_UPDATE_VISITED = 41 @@ -25,6 +27,7 @@ const ( NUM_NEWS = 10 DAYS_NEWS_INDEXPAGE = 15 + STORE_PATH = "store/" TEMPLATE_PATH = "templates/" CSS_PATH = "css/" JS_PATH = "js/" diff --git a/cover.go b/cover.go index 66be75b..6ebd342 100644 --- a/cover.go +++ b/cover.go @@ -8,15 +8,13 @@ import _ "image/gif" import ( "bytes" "git.gitorious.org/go-pkg/epubgo.git" - "git.gitorious.org/trantor/trantor.git/database" + "git.gitorious.org/trantor/trantor.git/storage" "github.com/gorilla/mux" "github.com/nfnt/resize" "image" "image/jpeg" "io" "io/ioutil" - "labix.org/v2/mgo" - "labix.org/v2/mgo/bson" "regexp" "strings" ) @@ -36,13 +34,11 @@ func coverHandler(h handler) { } } - fs := h.db.GetFS(FS_IMGS) - var f *mgo.GridFile + file := COVER_FILE if vars["size"] == "small" { - f, err = fs.OpenId(book.CoverSmall) - } else { - f, err = fs.OpenId(book.Cover) + file = COVER_SMALL_FILE } + f, err := h.store.Get(book.Id, file) if err != nil { log.Error("Error while opening image: ", err) notFound(h) @@ -53,18 +49,21 @@ func coverHandler(h handler) { headers := h.w.Header() headers["Content-Type"] = []string{"image/jpeg"} - io.Copy(h.w, f) + _, err = io.Copy(h.w, f) + if err != nil { + log.Error("Error while copying image: ", err) + notFound(h) + return + } } -func GetCover(e *epubgo.Epub, title string, db *database.DB) (bson.ObjectId, bson.ObjectId) { - imgId, smallId := coverFromMetadata(e, title, db) - if imgId != "" { - return imgId, smallId +func GetCover(e *epubgo.Epub, id string, store *storage.Store) bool { + if coverFromMetadata(e, id, store) { + return true } - imgId, smallId = searchCommonCoverNames(e, title, db) - if imgId != "" { - return imgId, smallId + if searchCommonCoverNames(e, id, store) { + return true } /* search for img on the text */ @@ -103,52 +102,52 @@ func GetCover(e *epubgo.Epub, title string, db *database.DB) (bson.ObjectId, bso img, err := e.OpenFile(url) if err == nil { defer img.Close() - return storeImg(img, title, db) + return storeImg(img, id, store) } } errNext = it.Next() } - return "", "" + return false } -func coverFromMetadata(e *epubgo.Epub, title string, db *database.DB) (bson.ObjectId, bson.ObjectId) { +func coverFromMetadata(e *epubgo.Epub, id string, store *storage.Store) bool { metaList, _ := e.MetadataAttr("meta") for _, meta := range metaList { if meta["name"] == "cover" { img, err := e.OpenFileId(meta["content"]) if err == nil { defer img.Close() - return storeImg(img, title, db) + return storeImg(img, id, store) } } } - return "", "" + return false } -func searchCommonCoverNames(e *epubgo.Epub, title string, db *database.DB) (bson.ObjectId, bson.ObjectId) { +func searchCommonCoverNames(e *epubgo.Epub, id string, store *storage.Store) bool { for _, p := range []string{"cover.jpg", "Images/cover.jpg", "images/cover.jpg", "cover.jpeg", "cover1.jpg", "cover1.jpeg"} { img, err := e.OpenFile(p) if err == nil { defer img.Close() - return storeImg(img, title, db) + return storeImg(img, id, store) } } - return "", "" + return false } -func storeImg(img io.Reader, title string, db *database.DB) (bson.ObjectId, bson.ObjectId) { +func storeImg(img io.Reader, id string, store *storage.Store) bool { /* open the files */ - fBig, err := createCoverFile(title, db) + fBig, err := store.Create(id, COVER_FILE) if err != nil { - log.Error("Error creating ", title, ": ", err.Error()) - return "", "" + log.Error("Error creating cover ", id, ": ", err.Error()) + return false } defer fBig.Close() - fSmall, err := createCoverFile(title+"_small", db) + fSmall, err := store.Create(id, COVER_SMALL_FILE) if err != nil { - log.Error("Error creating ", title+"_small", ": ", err.Error()) - return "", "" + log.Error("Error creating small cover ", id, ": ", err.Error()) + return false } defer fSmall.Close() @@ -159,32 +158,24 @@ func storeImg(img io.Reader, title string, db *database.DB) (bson.ObjectId, bson imgResized, err := resizeImg(img1, IMG_WIDTH_BIG) if err != nil { log.Error("Error resizing big image: ", err.Error()) - return "", "" + return false } err = jpeg.Encode(fBig, imgResized, &jpgOptions) if err != nil { log.Error("Error encoding big image: ", err.Error()) - return "", "" + return false } imgSmallResized, err := resizeImg(&img2, IMG_WIDTH_SMALL) if err != nil { log.Error("Error resizing small image: ", err.Error()) - return "", "" + return false } err = jpeg.Encode(fSmall, imgSmallResized, &jpgOptions) if err != nil { log.Error("Error encoding small image: ", err.Error()) - return "", "" + return false } - - idBig, _ := fBig.Id().(bson.ObjectId) - idSmall, _ := fSmall.Id().(bson.ObjectId) - return idBig, idSmall -} - -func createCoverFile(title string, db *database.DB) (*mgo.GridFile, error) { - fs := db.GetFS(FS_IMGS) - return fs.Create(title + ".jpg") + return true } func resizeImg(imgReader io.Reader, width uint) (image.Image, error) { diff --git a/database/books.go b/database/books.go index 701bdc8..6bef9d7 100644 --- a/database/books.go +++ b/database/books.go @@ -1,7 +1,6 @@ package database import ( - "errors" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "gopkgs.com/unidecode.v1" @@ -14,7 +13,7 @@ const ( ) type Book struct { - Id string `bson:"_id"` + Id string Title string Author []string Contributor string @@ -31,10 +30,8 @@ type Book struct { Coverage string Rights string Meta string - File bson.ObjectId FileSize int - Cover bson.ObjectId - CoverSmall bson.ObjectId + Cover bool Active bool Keywords []string } @@ -66,35 +63,41 @@ func _getBooks(coll *mgo.Collection, query bson.M, length int, start int) (books } err = q.All(&books) - for i, b := range books { - books[i].Id = bson.ObjectId(b.Id).Hex() - } return } func getBookId(coll *mgo.Collection, id string) (Book, error) { var book Book - if !bson.IsObjectIdHex(id) { - return book, errors.New("Not valid book id") - } - - err := coll.FindId(bson.ObjectIdHex(id)).One(&book) - book.Id = bson.ObjectId(book.Id).Hex() + err := coll.Find(bson.M{"id": id}).One(&book) return book, err } func deleteBook(coll *mgo.Collection, id string) error { - return coll.RemoveId(bson.ObjectIdHex(id)) + return coll.Remove(bson.M{"id": id}) } func updateBook(coll *mgo.Collection, id string, data map[string]interface{}) error { - data["keywords"] = keywords(data) - return coll.UpdateId(bson.ObjectIdHex(id), bson.M{"$set": data}) + var book map[string]interface{} + err := coll.Find(bson.M{"id": id}).One(&book) + if err != nil { + return err + } + for k, v := range data { + book[k] = v + } + + data["keywords"] = keywords(book) + return coll.Update(bson.M{"id": id}, bson.M{"$set": data}) } -func bookActive(coll *mgo.Collection, id string) bool { +func activeBook(coll *mgo.Collection, id string) error { + data := map[string]interface{}{"active": true} + return coll.Update(bson.M{"id": id}, bson.M{"$set": data}) +} + +func isBookActive(coll *mgo.Collection, id string) bool { var book Book - err := coll.FindId(bson.ObjectIdHex(id)).One(&book) + err := coll.Find(bson.M{"id": id}).One(&book) if err != nil { return false } @@ -123,15 +126,29 @@ func buildQuery(q string) bson.M { func keywords(b map[string]interface{}) (k []string) { title, _ := b["title"].(string) k = tokens(title) - author, _ := b["author"].([]string) - for _, a := range author { - k = append(k, tokens(a)...) - } + + k = append(k, listKeywords(b["author"])...) + publisher, _ := b["publisher"].(string) k = append(k, tokens(publisher)...) - subject, _ := b["subject"].([]string) - for _, s := range subject { - k = append(k, tokens(s)...) + + k = append(k, listKeywords(b["subject"])...) + return +} + +func listKeywords(v interface{}) (k []string) { + list, ok := v.([]string) + if !ok { + list, _ := v.([]interface{}) + for _, e := range list { + str := e.(string) + k = append(k, tokens(str)...) + } + return + } + + for _, e := range list { + k = append(k, tokens(e)...) } return } diff --git a/database/books_test.go b/database/books_test.go index fd9744d..a32e6dc 100644 --- a/database/books_test.go +++ b/database/books_test.go @@ -5,6 +5,7 @@ import "testing" var book = map[string]interface{}{ "title": "some title", "author": []string{"Alice", "Bob"}, + "id": "r_m-IOzzIbA6QK5w", } func TestAddBook(t *testing.T) { @@ -15,7 +16,7 @@ func TestAddBook(t *testing.T) { books, num, err := db.GetNewBooks(1, 0) if err != nil { - t.Fatalf("db.GetBooks() return an error: ", err) + t.Fatal("db.GetBooks() return an error: ", err) } if num < 1 { t.Fatalf("db.GetBooks() didn't find any result.") @@ -24,13 +25,61 @@ func TestAddBook(t *testing.T) { t.Fatalf("db.GetBooks() didn't return any result.") } if books[0].Title != book["title"] { - t.Errorf("Book title don't match : '", books[0].Title, "' <=> '", book["title"], "'") + t.Error("Book title don't match : '", books[0].Title, "' <=> '", book["title"], "'") + } +} + +func TestActiveBook(t *testing.T) { + db := Init(test_host, test_coll) + defer db.del() + + tAddBook(t, db) + books, _, _ := db.GetNewBooks(1, 0) + id := books[0].Id + + err := db.ActiveBook(id) + if err != nil { + t.Fatal("db.ActiveBook(", id, ") return an error: ", err) + } + + b, err := db.GetBookId(id) + if err != nil { + t.Fatal("db.GetBookId(", id, ") return an error: ", err) + } + if b.Author[0] != books[0].Author[0] { + t.Error("Book author don't match : '", b.Author, "' <=> '", book["author"], "'") + } +} + +func TestUpdateBookKeywords(t *testing.T) { + db := Init(test_host, test_coll) + defer db.del() + + tAddBook(t, db) + books, _, _ := db.GetNewBooks(1, 0) + + db.UpdateBook(books[0].Id, map[string]interface{}{"title": "Some other title"}) + books, _, _ = db.GetNewBooks(1, 0) + keywords := books[0].Keywords + + alice := false + bob := false + for _, e := range keywords { + if e == "alice" { + alice = true + } + if e == "bob" { + bob = true + } + } + if !alice || !bob { + t.Error("Alce or Bob are not in the keywords:", keywords) } } func tAddBook(t *testing.T, db *DB) { err := db.AddBook(book) if err != nil { - t.Errorf("db.AddBook(", book, ") return an error: ", err) + t.Error("db.AddBook(", book, ") return an error: ", err) } } diff --git a/database/database.go b/database/database.go index dad79a7..1b32c68 100644 --- a/database/database.go +++ b/database/database.go @@ -73,9 +73,14 @@ func (db *DB) UpdateBook(id string, data map[string]interface{}) error { return updateBook(booksColl, id, data) } -func (db *DB) BookActive(id string) bool { +func (db *DB) ActiveBook(id string) error { booksColl := db.session.DB(db.name).C(books_coll) - return bookActive(booksColl, id) + return activeBook(booksColl, id) +} + +func (db *DB) IsBookActive(id string) bool { + booksColl := db.session.DB(db.name).C(books_coll) + return isBookActive(booksColl, id) } func (db *DB) User(name string) *User { @@ -154,10 +159,6 @@ func (db *DB) UpdateDownloadedBooks() error { return u.UpdateMostBooks("download") } -func (db *DB) GetFS(prefix string) *mgo.GridFS { - return db.session.DB(db.name).GridFS(prefix) -} - func (db *DB) GetTags() ([]string, error) { tagsColl := db.session.DB(db.name).C(tags_coll) return GetTags(tagsColl) diff --git a/reader.go b/reader.go index 5402076..ad2b9ce 100644 --- a/reader.go +++ b/reader.go @@ -1,11 +1,15 @@ package main +import log "github.com/cihub/seelog" + import ( + "bytes" "git.gitorious.org/go-pkg/epubgo.git" "git.gitorious.org/trantor/trantor.git/database" + "git.gitorious.org/trantor/trantor.git/storage" "github.com/gorilla/mux" "io" - "labix.org/v2/mgo/bson" + "io/ioutil" "net/http" "strconv" "strings" @@ -134,6 +138,7 @@ func readStartHandler(h handler) { id := mux.Vars(h.r)["id"] e, _ := openReadEpub(h) if e == nil { + log.Warn("Open epub returns an empty file") notFound(h) return } @@ -141,6 +146,7 @@ func readStartHandler(h handler) { it, err := e.Spine() if err != nil { + log.Warn("No spine in the epub") notFound(h) return } @@ -175,7 +181,7 @@ func readHandler(h handler) { func openReadEpub(h handler) (*epubgo.Epub, database.Book) { var book database.Book id := mux.Vars(h.r)["id"] - if !bson.IsObjectIdHex(id) { + if id == "" { return nil, book } book, err := h.db.GetBookId(id) @@ -188,7 +194,7 @@ func openReadEpub(h handler) (*epubgo.Epub, database.Book) { return nil, book } } - e, err := OpenBook(book.File, h.db) + e, err := openBook(book.Id, h.store) if err != nil { return nil, book } @@ -199,7 +205,7 @@ func contentHandler(h handler) { vars := mux.Vars(h.r) id := vars["id"] file := vars["file"] - if file == "" || !bson.IsObjectIdHex(id) { + if file == "" || id == "" { notFound(h) return } @@ -215,7 +221,7 @@ func contentHandler(h handler) { return } } - e, err := OpenBook(book.File, h.db) + e, err := openBook(book.Id, h.store) if err != nil { notFound(h) return @@ -230,3 +236,16 @@ func contentHandler(h handler) { defer html.Close() io.Copy(h.w, html) } + +func openBook(id string, store *storage.Store) (*epubgo.Epub, error) { + f, err := store.Get(id, EPUB_FILE) + if err != nil { + return nil, err + } + defer f.Close() + + buff, err := ioutil.ReadAll(f) + reader := bytes.NewReader(buff) + + return epubgo.Load(reader, int64(len(buff))) +} diff --git a/stats.go b/stats.go index e2552c7..14ef110 100644 --- a/stats.go +++ b/stats.go @@ -4,32 +4,49 @@ import log "github.com/cihub/seelog" import ( "git.gitorious.org/trantor/trantor.git/database" + "git.gitorious.org/trantor/trantor.git/storage" "github.com/gorilla/mux" - "labix.org/v2/mgo/bson" "net/http" "strconv" "strings" "time" ) +const ( + stats_version = 2 +) + type handler struct { - w http.ResponseWriter - r *http.Request - sess *Session - db *database.DB + w http.ResponseWriter + r *http.Request + sess *Session + db *database.DB + store *storage.Store } -func InitStats(database *database.DB) { - statsChannel = make(chan statsRequest, CHAN_SIZE) - go statsWorker(database) +type StatsGatherer struct { + db *database.DB + store *storage.Store + channel chan statsRequest } -func GatherStats(function func(handler), database *database.DB) func(http.ResponseWriter, *http.Request) { +func InitStats(database *database.DB, store *storage.Store) *StatsGatherer { + sg := new(StatsGatherer) + sg.channel = make(chan statsRequest, CHAN_SIZE) + sg.db = database + sg.store = store + + go sg.worker() + return sg +} + +func (sg StatsGatherer) Gather(function func(handler)) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, r *http.Request) { log.Info("Query ", r.Method, " ", r.RequestURI) var h handler - h.db = database.Copy() + h.store = sg.store + h.db = sg.db.Copy() defer h.db.Close() h.w = w @@ -37,12 +54,10 @@ func GatherStats(function func(handler), database *database.DB) func(http.Respon h.sess = GetSession(r, h.db) function(h) - statsChannel <- statsRequest{bson.Now(), mux.Vars(r), h.sess, r} + sg.channel <- statsRequest{time.Now(), mux.Vars(r), h.sess, r} } } -var statsChannel chan statsRequest - type statsRequest struct { date time.Time vars map[string]string @@ -50,16 +65,17 @@ type statsRequest struct { r *http.Request } -func statsWorker(database *database.DB) { - db := database.Copy() +func (sg StatsGatherer) worker() { + db := sg.db.Copy() defer db.Close() - for req := range statsChannel { + for req := range sg.channel { stats := make(map[string]interface{}) appendFiles(req.r, stats) appendMuxVars(req.vars, stats) appendUrl(req.r, stats) appendSession(req.sess, stats) + stats["version"] = stats_version stats["method"] = req.r.Method stats["date"] = req.date db.AddStats(stats) @@ -141,16 +157,12 @@ func appendMuxVars(vars map[string]string, stats map[string]interface{}) { for key, value := range vars { switch { case key == "id": - if bson.IsObjectIdHex(value) { - stats["id"] = bson.ObjectIdHex(value) - } + stats["id"] = value case key == "ids": - var objectIds []bson.ObjectId + var objectIds []string ids := strings.Split(value, "/") for _, id := range ids { - if bson.IsObjectIdHex(value) { - objectIds = append(objectIds, bson.ObjectIdHex(id)) - } + objectIds = append(objectIds, id) } if len(objectIds) > 0 { stats["ids"] = objectIds diff --git a/storage/dir.go b/storage/dir.go new file mode 100644 index 0000000..066f0bb --- /dev/null +++ b/storage/dir.go @@ -0,0 +1,42 @@ +package storage + +import p "path" + +import ( + "os" +) + +const ( + dir_depth = 2 + encodeURL = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" +) + +func mkstore(path string) error { + return _mkstore(path, dir_depth) +} + +func _mkstore(path string, depth int) error { + err := os.MkdirAll(path, os.ModePerm) + if err != nil || depth == 0 { + return err + } + + for _, l := range encodeURL { + next_path := p.Join(path, string(l)) + err = _mkstore(next_path, depth-1) + if err != nil { + return err + } + } + return nil +} + +func idPath(storePath string, id string) string { + path := storePath + for i := 0; i < dir_depth; i++ { + dir := string(id[i]) + path = p.Join(path, dir) + } + path = p.Join(path, id) + return path +} diff --git a/storage/storage.go b/storage/storage.go new file mode 100644 index 0000000..f84b3ab --- /dev/null +++ b/storage/storage.go @@ -0,0 +1,57 @@ +package storage + +import p "path" + +import ( + "io" + "os" +) + +type Store struct { + path string +} + +func Init(path string) (*Store, error) { + st := new(Store) + st.path = path + + _, err := os.Stat(path) + if err != nil { + err = mkstore(st.path) + } + return st, err +} + +func (st *Store) Create(id string, name string) (io.WriteCloser, error) { + path := idPath(st.path, id) + err := os.MkdirAll(path, os.ModePerm) + if err != nil { + return nil, err + } + + return os.Create(p.Join(path, name)) +} + +func (st *Store) Store(id string, file io.Reader, name string) (size int64, err error) { + dest, err := st.Create(id, name) + if err != nil { + return 0, err + } + defer dest.Close() + + return io.Copy(dest, file) +} + +func (st *Store) Get(id string, name string) (io.ReadCloser, error) { + path := idPath(st.path, id) + return os.Open(p.Join(path, name)) +} + +func (st *Store) Delete(id string) error { + path := idPath(st.path, id) + return os.RemoveAll(path) +} + +func (st *Store) del() { + os.RemoveAll(st.path) +} diff --git a/storage/storage_test.go b/storage/storage_test.go new file mode 100644 index 0000000..c8aa657 --- /dev/null +++ b/storage/storage_test.go @@ -0,0 +1,113 @@ +package storage + +import "testing" + +import ( + "bytes" + "io" + "io/ioutil" + "os" + "strings" +) + +const ( + test_path = "/tmp/store" + test_book = ` + HARI SELDON-… born in the 11,988th year of the Galactic Era; died + 12,069. The dates are more commonly given in terms of the current + Foundational Era as - 79 to the year 1 F.E. Born to middle-class + parents on Helicon, Arcturus sector (where his father, in a legend of + doubtful authenticity, was a tobacco grower in the hydroponic plants + of the planet), he early showed amazing ability in mathematics. + Anecdotes concerning his ability are innumerable, and some are + contradictory. At the age of two, he is said to have …` + test_id = "1234567890abcdef" +) + +func TestInit(t *testing.T) { + st, err := Init(test_path) + if err != nil { + t.Fatal("An error ocurred initializing the store =>", err) + } + defer st.del() + + info, err := os.Stat(test_path) + if err != nil { + t.Fatal("An error ocurred =>", err) + } + if !info.Mode().IsDir() { + t.Errorf(test_path, " is not dir.") + } + + info, err = os.Stat(test_path + "/a/M") + if err != nil { + t.Fatal("An error ocurred =>", err) + } + if !info.Mode().IsDir() { + t.Errorf(test_path, " is not dir.") + } +} + +func TestStore(t *testing.T) { + st, err := Init(test_path) + defer st.del() + + _, err = st.Store(test_id, strings.NewReader(test_book), "epub") + if err != nil { + t.Fatal("An error ocurred storing the book =>", err) + } + book, err := st.Get(test_id, "epub") + if err != nil { + t.Fatal("An error ocurred getting the book =>", err) + } + + content, err := ioutil.ReadAll(book) + if err != nil { + t.Fatal("An error ocurred reading the book =>", err) + } + if !bytes.Equal(content, []byte(test_book)) { + t.Error("Not the same content") + } +} + +func TestCreate(t *testing.T) { + st, err := Init(test_path) + defer st.del() + + f, err := st.Create(test_id, "img") + if err != nil { + t.Fatal("An error ocurred storing the book =>", err) + } + io.Copy(f, strings.NewReader(test_book)) + img, err := st.Get(test_id, "img") + if err != nil { + t.Fatal("An error ocurred getting the book =>", err) + } + + content, err := ioutil.ReadAll(img) + if err != nil { + t.Fatal("An error ocurred reading the book =>", err) + } + if !bytes.Equal(content, []byte(test_book)) { + t.Error("Not the same content") + } +} + +func TestDelete(t *testing.T) { + st, err := Init(test_path) + defer st.del() + + _, err = st.Store(test_id, strings.NewReader(test_book), "epub") + if err != nil { + t.Fatal("An error ocurred storing the book =>", err) + } + err = st.Delete(test_id) + if err != nil { + t.Fatal("An error ocurred deleteing id =>", err) + } + + _, err = st.Get(test_id, "epub") + if err == nil { + t.Fatal("Retrieve book without error.") + } +} diff --git a/store.go b/store.go deleted file mode 100644 index 2a8f1f7..0000000 --- a/store.go +++ /dev/null @@ -1,115 +0,0 @@ -package main - -import ( - "bytes" - "git.gitorious.org/go-pkg/epubgo.git" - "git.gitorious.org/trantor/trantor.git/database" - "io" - "io/ioutil" - "labix.org/v2/mgo/bson" - "regexp" - "strings" -) - -func OpenBook(id bson.ObjectId, db *database.DB) (*epubgo.Epub, error) { - fs := db.GetFS(FS_BOOKS) - f, err := fs.OpenId(id) - if err != nil { - return nil, err - } - defer f.Close() - - buff, err := ioutil.ReadAll(f) - reader := bytes.NewReader(buff) - - return epubgo.Load(reader, int64(len(buff))) -} - -func StoreNewFile(name string, file io.Reader, db *database.DB) (bson.ObjectId, int64, error) { - fs := db.GetFS(FS_BOOKS) - fw, err := fs.Create(name) - if err != nil { - return "", 0, err - } - defer fw.Close() - - size, err := io.Copy(fw, file) - id, _ := fw.Id().(bson.ObjectId) - return id, size, err -} - -func DeleteFile(id bson.ObjectId, db *database.DB) error { - fs := db.GetFS(FS_BOOKS) - return fs.RemoveId(id) -} - -func DeleteCover(id bson.ObjectId, db *database.DB) error { - fs := db.GetFS(FS_IMGS) - return fs.RemoveId(id) -} - -func DeleteBook(book database.Book, db *database.DB) { - if book.Cover != "" { - DeleteCover(book.Cover, db) - } - if book.CoverSmall != "" { - DeleteCover(book.CoverSmall, db) - } - DeleteFile(book.File, db) -} - -func cleanStr(str string) string { - str = strings.Replace(str, "'", "'", -1) - exp, _ := regexp.Compile("&[^;]*;") - str = exp.ReplaceAllString(str, "") - exp, _ = regexp.Compile("[ ,]*$") - str = exp.ReplaceAllString(str, "") - return str -} - -func parseAuthr(creator []string) []string { - exp1, _ := regexp.Compile("^(.*\\( *([^\\)]*) *\\))*$") - exp2, _ := regexp.Compile("^[^:]*: *(.*)$") - res := make([]string, len(creator)) - for i, s := range creator { - auth := exp1.FindStringSubmatch(s) - if auth != nil { - res[i] = cleanStr(strings.Join(auth[2:], ", ")) - } else { - auth := exp2.FindStringSubmatch(s) - if auth != nil { - res[i] = cleanStr(auth[1]) - } else { - res[i] = cleanStr(s) - } - } - } - return res -} - -func parseDescription(description []string) string { - str := cleanStr(strings.Join(description, "\n")) - str = strings.Replace(str, "

", "\n", -1) - exp, _ := regexp.Compile("<[^>]*>") - str = exp.ReplaceAllString(str, "") - str = strings.Replace(str, "&", "&", -1) - str = strings.Replace(str, "<", "<", -1) - str = strings.Replace(str, ">", ">", -1) - str = strings.Replace(str, "\\n", "\n", -1) - return str -} - -func parseSubject(subject []string) []string { - var res []string - for _, s := range subject { - res = append(res, strings.Split(s, " / ")...) - } - return res -} - -func parseDate(date []string) string { - if len(date) == 0 { - return "" - } - return strings.Replace(date[0], "Unspecified: ", "", -1) -} diff --git a/templates/index.html b/templates/index.html index 6a41dd6..e3cfcd2 100644 --- a/templates/index.html +++ b/templates/index.html @@ -26,7 +26,7 @@
  • - {{if .CoverSmall}}
    {{.Title}}
    {{end}} + {{if .Cover}}
    {{.Title}}
    {{end}}

    {{.Title}}
    @@ -44,7 +44,7 @@
  • - {{if .CoverSmall}}
    {{.Title}}
    {{end}} + {{if .Cover}}
    {{.Title}}
    {{end}}

    {{.Title}}
    @@ -62,7 +62,7 @@
  • - {{if .CoverSmall}}
    {{.Title}}
    {{end}} + {{if .Cover}}
    {{.Title}}
    {{end}}

    {{.Title}}
    diff --git a/templates/new.html b/templates/new.html index 58756bc..50c0a29 100644 --- a/templates/new.html +++ b/templates/new.html @@ -26,7 +26,7 @@ {{with .B}}
    -

    {{if .CoverSmall}}{{.Title}}{{end}}

    +

    {{if .Cover}}{{.Title}}{{end}}

    {{.Title}} ({{$titleFound}})
    diff --git a/templates/search.html b/templates/search.html index b1a9d6c..1d191d1 100644 --- a/templates/search.html +++ b/templates/search.html @@ -21,7 +21,7 @@ {{range .}}

    -

    {{if .CoverSmall}}{{.Title}}{{end}}

    +

    {{if .Cover}}{{.Title}}{{end}}

    diff --git a/trantor.go b/trantor.go index 533f620..38f1db7 100644 --- a/trantor.go +++ b/trantor.go @@ -4,9 +4,11 @@ import log "github.com/cihub/seelog" import ( "git.gitorious.org/trantor/trantor.git/database" + "git.gitorious.org/trantor/trantor.git/storage" "github.com/gorilla/mux" "io" "net/http" + "os" "strings" ) @@ -71,8 +73,7 @@ func downloadHandler(h handler) { } } - fs := h.db.GetFS(FS_BOOKS) - f, err := fs.OpenId(book.File) + f, err := h.store.Get(book.Id, EPUB_FILE) if err != nil { notFound(h) return @@ -81,7 +82,7 @@ func downloadHandler(h handler) { headers := h.w.Header() headers["Content-Type"] = []string{"application/epub+zip"} - headers["Content-Disposition"] = []string{"attachment; filename=\"" + f.Name() + "\""} + headers["Content-Disposition"] = []string{"attachment; filename=\"" + book.Title + ".epub\""} io.Copy(h.w, f) } @@ -137,47 +138,55 @@ func main() { db := database.Init(DB_IP, DB_NAME) defer db.Close() - InitTasks(db) - InitStats(db) - InitUpload(db) + store, err := storage.Init(STORE_PATH) + if err != nil { + log.Critical("Problem initializing store: ", err) + os.Exit(1) + } - initRouter(db) + InitTasks(db) + sg := InitStats(db, store) + InitUpload(db, store) + + initRouter(db, sg) log.Error(http.ListenAndServe(":"+PORT, nil)) } -func initRouter(db *database.DB) { +func initRouter(db *database.DB, sg *StatsGatherer) { + const id_pattern = "[0-9a-zA-Z\\-\\_]{16}" + r := mux.NewRouter() var notFoundHandler http.HandlerFunc - notFoundHandler = GatherStats(notFound, db) + notFoundHandler = sg.Gather(notFound) r.NotFoundHandler = notFoundHandler - r.HandleFunc("/", GatherStats(indexHandler, db)) - r.HandleFunc("/book/{id:[0-9a-fA-F]+}", GatherStats(bookHandler, db)) - r.HandleFunc("/search/", GatherStats(searchHandler, db)) - r.HandleFunc("/upload/", GatherStats(uploadHandler, db)).Methods("GET") - r.HandleFunc("/upload/", GatherStats(uploadPostHandler, db)).Methods("POST") - r.HandleFunc("/login/", GatherStats(loginHandler, db)).Methods("GET") - r.HandleFunc("/login/", GatherStats(loginPostHandler, db)).Methods("POST") - r.HandleFunc("/create_user/", GatherStats(createUserHandler, db)).Methods("POST") - r.HandleFunc("/logout/", GatherStats(logoutHandler, db)) - r.HandleFunc("/new/", GatherStats(newHandler, db)) - r.HandleFunc("/store/{ids:([0-9a-fA-F]+/)+}", GatherStats(storeHandler, db)) - r.HandleFunc("/delete/{ids:([0-9a-fA-F]+/)+}", GatherStats(deleteHandler, db)) - r.HandleFunc("/read/{id:[0-9a-fA-F]+}", GatherStats(readStartHandler, db)) - r.HandleFunc("/read/{id:[0-9a-fA-F]+}/{file:.*}", GatherStats(readHandler, db)) - r.HandleFunc("/content/{id:[0-9a-fA-F]+}/{file:.*}", GatherStats(contentHandler, db)) - r.HandleFunc("/edit/{id:[0-9a-fA-F]+}", GatherStats(editHandler, db)) - r.HandleFunc("/save/{id:[0-9a-fA-F]+}", GatherStats(saveHandler, db)).Methods("POST") - r.HandleFunc("/about/", GatherStats(aboutHandler, db)) - r.HandleFunc("/help/", GatherStats(helpHandler, db)) - r.HandleFunc("/download/{id:[0-9a-fA-F]+}/{epub:.*}", GatherStats(downloadHandler, db)) - r.HandleFunc("/cover/{id:[0-9a-fA-F]+}/{size}/{img:.*}", GatherStats(coverHandler, db)) - r.HandleFunc("/dashboard/", GatherStats(dashboardHandler, db)) - r.HandleFunc("/settings/", GatherStats(settingsHandler, db)) - r.HandleFunc("/stats/", GatherStats(statsHandler, db)) - r.HandleFunc("/news/", GatherStats(newsHandler, db)) - r.HandleFunc("/news/edit", GatherStats(editNewsHandler, db)).Methods("GET") - r.HandleFunc("/news/edit", GatherStats(postNewsHandler, db)).Methods("POST") + r.HandleFunc("/", sg.Gather(indexHandler)) + r.HandleFunc("/book/{id:"+id_pattern+"}", sg.Gather(bookHandler)) + r.HandleFunc("/search/", sg.Gather(searchHandler)) + r.HandleFunc("/upload/", sg.Gather(uploadHandler)).Methods("GET") + r.HandleFunc("/upload/", sg.Gather(uploadPostHandler)).Methods("POST") + r.HandleFunc("/login/", sg.Gather(loginHandler)).Methods("GET") + r.HandleFunc("/login/", sg.Gather(loginPostHandler)).Methods("POST") + r.HandleFunc("/create_user/", sg.Gather(createUserHandler)).Methods("POST") + r.HandleFunc("/logout/", sg.Gather(logoutHandler)) + r.HandleFunc("/new/", sg.Gather(newHandler)) + r.HandleFunc("/store/{ids:("+id_pattern+"/)+}", sg.Gather(storeHandler)) + r.HandleFunc("/delete/{ids:("+id_pattern+"/)+}", sg.Gather(deleteHandler)) + r.HandleFunc("/read/{id:"+id_pattern+"}", sg.Gather(readStartHandler)) + r.HandleFunc("/read/{id:"+id_pattern+"}/{file:.*}", sg.Gather(readHandler)) + r.HandleFunc("/content/{id:"+id_pattern+"}/{file:.*}", sg.Gather(contentHandler)) + r.HandleFunc("/edit/{id:"+id_pattern+"}", sg.Gather(editHandler)) + r.HandleFunc("/save/{id:"+id_pattern+"}", sg.Gather(saveHandler)).Methods("POST") + r.HandleFunc("/about/", sg.Gather(aboutHandler)) + r.HandleFunc("/help/", sg.Gather(helpHandler)) + r.HandleFunc("/download/{id:"+id_pattern+"}/{epub:.*}", sg.Gather(downloadHandler)) + r.HandleFunc("/cover/{id:"+id_pattern+"}/{size}/{img:.*}", sg.Gather(coverHandler)) + r.HandleFunc("/dashboard/", sg.Gather(dashboardHandler)) + r.HandleFunc("/settings/", sg.Gather(settingsHandler)) + r.HandleFunc("/stats/", sg.Gather(statsHandler)) + r.HandleFunc("/news/", sg.Gather(newsHandler)) + r.HandleFunc("/news/edit", sg.Gather(editNewsHandler)).Methods("GET") + r.HandleFunc("/news/edit", sg.Gather(postNewsHandler)).Methods("POST") h := http.FileServer(http.Dir(IMG_PATH)) r.Handle("/img/{img}", http.StripPrefix("/img/", h)) h = http.FileServer(http.Dir(CSS_PATH)) diff --git a/upload.go b/upload.go index 832e0da..ceb4306 100644 --- a/upload.go +++ b/upload.go @@ -4,16 +4,20 @@ import log "github.com/cihub/seelog" import ( "bytes" + "crypto/rand" + "encoding/base64" "git.gitorious.org/go-pkg/epubgo.git" "git.gitorious.org/trantor/trantor.git/database" + "git.gitorious.org/trantor/trantor.git/storage" "io/ioutil" "mime/multipart" + "regexp" "strings" ) -func InitUpload(database *database.DB) { +func InitUpload(database *database.DB, store *storage.Store) { uploadChannel = make(chan uploadRequest, CHAN_SIZE) - go uploadWorker(database) + go uploadWorker(database, store) } var uploadChannel chan uploadRequest @@ -23,16 +27,16 @@ type uploadRequest struct { filename string } -func uploadWorker(database *database.DB) { +func uploadWorker(database *database.DB, store *storage.Store) { db := database.Copy() defer db.Close() for req := range uploadChannel { - processFile(req, db) + processFile(req, db, store) } } -func processFile(req uploadRequest, db *database.DB) { +func processFile(req uploadRequest, db *database.DB, store *storage.Store) { defer req.file.Close() epub, err := openMultipartEpub(req.file) @@ -42,20 +46,18 @@ func processFile(req uploadRequest, db *database.DB) { } defer epub.Close() - book := parseFile(epub, db) - title, _ := book["title"].(string) + book, id := parseFile(epub, store) req.file.Seek(0, 0) - id, size, err := StoreNewFile(title+".epub", req.file, db) + size, err := store.Store(id, req.file, EPUB_FILE) if err != nil { - log.Error("Error storing book (", title, "): ", err) + log.Error("Error storing book (", id, "): ", err) return } - book["file"] = id book["filesize"] = size err = db.AddBook(book) if err != nil { - log.Error("Error storing metadata (", title, "): ", err) + log.Error("Error storing metadata (", id, "): ", err) return } log.Info("File uploaded: ", req.filename) @@ -104,7 +106,7 @@ func openMultipartEpub(file multipart.File) (*epubgo.Epub, error) { return epubgo.Load(reader, int64(len(buff))) } -func parseFile(epub *epubgo.Epub, db *database.DB) map[string]interface{} { +func parseFile(epub *epubgo.Epub, store *storage.Store) (metadata map[string]interface{}, id string) { book := map[string]interface{}{} for _, m := range epub.MetadataFields() { data, err := epub.Metadata(m) @@ -135,12 +137,71 @@ func parseFile(epub *epubgo.Epub, db *database.DB) map[string]interface{} { book[m] = strings.Join(data, ", ") } } - title, _ := book["title"].(string) - book["file"] = nil - cover, coverSmall := GetCover(epub, title, db) - if cover != "" { - book["cover"] = cover - book["coversmall"] = coverSmall - } - return book + + id = genId() + book["id"] = id //TODO + book["cover"] = GetCover(epub, id, store) + return book, id +} + +func genId() string { + b := make([]byte, 12) + rand.Read(b) + return base64.URLEncoding.EncodeToString(b) +} + +func cleanStr(str string) string { + str = strings.Replace(str, "'", "'", -1) + exp, _ := regexp.Compile("&[^;]*;") + str = exp.ReplaceAllString(str, "") + exp, _ = regexp.Compile("[ ,]*$") + str = exp.ReplaceAllString(str, "") + return str +} + +func parseAuthr(creator []string) []string { + exp1, _ := regexp.Compile("^(.*\\( *([^\\)]*) *\\))*$") + exp2, _ := regexp.Compile("^[^:]*: *(.*)$") + res := make([]string, len(creator)) + for i, s := range creator { + auth := exp1.FindStringSubmatch(s) + if auth != nil { + res[i] = cleanStr(strings.Join(auth[2:], ", ")) + } else { + auth := exp2.FindStringSubmatch(s) + if auth != nil { + res[i] = cleanStr(auth[1]) + } else { + res[i] = cleanStr(s) + } + } + } + return res +} + +func parseDescription(description []string) string { + str := cleanStr(strings.Join(description, "\n")) + str = strings.Replace(str, "

    ", "\n", -1) + exp, _ := regexp.Compile("<[^>]*>") + str = exp.ReplaceAllString(str, "") + str = strings.Replace(str, "&", "&", -1) + str = strings.Replace(str, "<", "<", -1) + str = strings.Replace(str, ">", ">", -1) + str = strings.Replace(str, "\\n", "\n", -1) + return str +} + +func parseSubject(subject []string) []string { + var res []string + for _, s := range subject { + res = append(res, strings.Split(s, " / ")...) + } + return res +} + +func parseDate(date []string) string { + if len(date) == 0 { + return "" + } + return strings.Replace(date[0], "Unspecified: ", "", -1) }