From 05b641201cfaad1fd8a416fc160fb550790a9d6e Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Fri, 12 Apr 2013 01:05:40 +0200 Subject: [PATCH] Store files on GridFS --- admin.go | 3 +- config.go | 3 +- database.go | 10 ++++-- reader.go | 20 ++++++------ store.go | 76 +++++++++++++++++++++++---------------------- templates/book.html | 2 +- trantor.go | 29 ++++++++++++----- upload.go | 26 ++++++++-------- 8 files changed, 94 insertions(+), 75 deletions(-) diff --git a/admin.go b/admin.go index 31638dd..7060430 100644 --- a/admin.go +++ b/admin.go @@ -235,13 +235,12 @@ func storeHandler(w http.ResponseWriter, r *http.Request) { continue } book := books[0] - path, err := StoreBook(book) if err != nil { sess.Notify("An error ocurred!", err.Error(), "error") log.Println("Error storing book '", book.Title, "': ", err.Error()) continue } - db.UpdateBook(id, bson.M{"active": true, "path": path}) + db.UpdateBook(id, bson.M{"active": true}) titles = append(titles, book.Title) } if titles != nil { diff --git a/config.go b/config.go index 8318161..d55d409 100644 --- a/config.go +++ b/config.go @@ -9,6 +9,8 @@ const ( BOOKS_COLL = "books" TAGS_COLL = "tags" USERS_COLL = "users" + FS_BOOKS = "fs_books" + FS_IMGS = "fs_imgs" PASS_SALT = "ImperialLibSalt" MINUTES_UPDATE_TAGS = 10 @@ -17,7 +19,6 @@ const ( NEW_ITEMS_PAGE = 50 TEMPLATE_PATH = "templates/" - BOOKS_PATH = "books/" COVER_PATH = "cover/" NEW_PATH = "new/" CSS_PATH = "css/" diff --git a/database.go b/database.go index 5602dc7..3d4062c 100644 --- a/database.go +++ b/database.go @@ -31,7 +31,7 @@ type Book struct { Coverage string Rights string Meta string - Path string + File bson.ObjectId Cover string CoverSmall string Active bool @@ -102,8 +102,8 @@ func (d *DB) IncVisit(id bson.ObjectId) error { return d.books.Update(bson.M{"_id": id}, bson.M{"$inc": bson.M{"VisitsCount": 1}}) } -func (d *DB) IncDownload(path string) error { - return d.books.Update(bson.M{"path": path}, bson.M{"$inc": bson.M{"DownloadCount": 1}}) +func (d *DB) IncDownload(id bson.ObjectId) error { + return d.books.Update(bson.M{"_id": id}, bson.M{"$inc": bson.M{"DownloadCount": 1}}) } /* optional parameters: length and start index @@ -178,6 +178,10 @@ func (d *DB) BookActive(id bson.ObjectId) bool { return book.Active } +func (d *DB) GetFS(prefix string) *mgo.GridFS { + return d.session.DB(DB_NAME).GridFS(prefix) +} + func (d *DB) areTagsOutdated() bool { var result struct { Id bson.ObjectId `bson:"_id"` diff --git a/reader.go b/reader.go index 0aff43c..31ca9c5 100644 --- a/reader.go +++ b/reader.go @@ -149,7 +149,6 @@ func readHandler(w http.ResponseWriter, r *http.Request) { var data readData data.Book = books[0] - var bookPath string if !data.Book.Active { sess := GetSession(r) if sess.User == "" { @@ -157,12 +156,10 @@ func readHandler(w http.ResponseWriter, r *http.Request) { return } data.Back = "/new/" - bookPath = NEW_PATH + data.Book.Path } else { data.Back = "/book/" + id - bookPath = BOOKS_PATH + data.Book.Path } - e, err := epubgo.Open(bookPath) + e, err := OpenBook(data.Book.File) if err != nil { http.NotFound(w, r) return @@ -187,29 +184,30 @@ func readHandler(w http.ResponseWriter, r *http.Request) { func contentHandler(w http.ResponseWriter, r *http.Request) { _, id, file := parseUrl(r.URL.Path) + if file == "" { + http.NotFound(w, r) + return + } + books, _, err := db.GetBooks(bson.M{"_id": bson.ObjectIdHex(id)}) if err != nil || len(books) == 0 { http.NotFound(w, r) return } book := books[0] - var bookPath string if !book.Active { sess := GetSession(r) if sess.User == "" { http.NotFound(w, r) return } - bookPath = NEW_PATH + book.Path - } else { - bookPath = BOOKS_PATH + book.Path } - e, _ := epubgo.Open(bookPath) - defer e.Close() - if file == "" { + e, err := OpenBook(book.File) + if err != nil { http.NotFound(w, r) return } + defer e.Close() html, err := e.OpenFile(file) if err != nil { diff --git a/store.go b/store.go index c0e54c3..06a9955 100644 --- a/store.go +++ b/store.go @@ -3,19 +3,19 @@ package main import ( "git.gitorious.org/go-pkg/epubgo.git" "io" - "log" + "labix.org/v2/mgo" + "labix.org/v2/mgo/bson" "os" - "os/exec" "regexp" "strconv" "strings" "unicode/utf8" ) -func ParseFile(path string) (string, error) { +func ParseFile(id bson.ObjectId) (string, error) { book := map[string]interface{}{} - e, err := epubgo.Open(NEW_PATH + path) + e, err := OpenBook(id) if err != nil { return "", err } @@ -51,7 +51,7 @@ func ParseFile(path string) (string, error) { } } title, _ := book["title"].(string) - book["path"] = path + book["file"] = id cover, coverSmall := GetCover(e, title) book["cover"] = cover book["coversmall"] = coverSmall @@ -61,35 +61,47 @@ func ParseFile(path string) (string, error) { return title, nil } -func StoreNewFile(name string, file io.Reader) (string, error) { - path := storePath(name) - fw, err := os.Create(NEW_PATH + path) +func OpenBook(id bson.ObjectId) (*epubgo.Epub, error) { + fs := db.GetFS(FS_BOOKS) + var reader readerGrid + var err error + reader.f, err = fs.OpenId(id) + if err != nil { + return nil, err + } + defer reader.f.Close() + return epubgo.Load(reader, reader.f.Size()) +} + +type readerGrid struct { + f *mgo.GridFile +} + +func (r readerGrid) ReadAt(p []byte, off int64) (n int, err error) { + _, err = r.f.Seek(off, 0) + if err != nil { + return + } + + return r.f.Read(p) +} + +func StoreNewFile(name string, file io.Reader) (bson.ObjectId, error) { + fs := db.GetFS(FS_BOOKS) + fw, err := fs.Create(name) if err != nil { return "", err } defer fw.Close() _, err = io.Copy(fw, file) - return path, err + id, _ := fw.Id().(bson.ObjectId) + return id, err } -func StoreBook(book Book) (path string, err error) { - title := book.Title - path = validFileName(BOOKS_PATH, title, ".epub") - - oldPath := NEW_PATH + book.Path - r, _ := utf8.DecodeRuneInString(title) - folder := string(r) - if _, err = os.Stat(BOOKS_PATH + folder); err != nil { - err = os.Mkdir(BOOKS_PATH+folder, os.ModePerm) - if err != nil { - log.Println("Error creating", BOOKS_PATH+folder, ":", err.Error()) - return - } - } - cmd := exec.Command("mv", oldPath, BOOKS_PATH+path) - err = cmd.Run() - return +func DeleteFile(id bson.ObjectId) error { + fs := db.GetFS(FS_BOOKS) + return fs.RemoveId(id) } func DeleteBook(book Book) { @@ -99,7 +111,7 @@ func DeleteBook(book Book) { if book.CoverSmall != "" { os.RemoveAll(book.CoverSmall[1:]) } - os.RemoveAll(book.Path) + DeleteFile(book.File) } func validFileName(path string, title string, extension string) string { @@ -117,16 +129,6 @@ func validFileName(path string, title string, extension string) string { return file } -func storePath(name string) string { - path := name - _, err := os.Stat(NEW_PATH + path) - for i := 0; err == nil; i++ { - path = strconv.Itoa(i) + "_" + name - _, err = os.Stat(NEW_PATH + path) - } - return path -} - func cleanStr(str string) string { str = strings.Replace(str, "'", "'", -1) exp, _ := regexp.Compile("&[^;]*;") diff --git a/templates/book.html b/templates/book.html index da55144..725e940 100644 --- a/templates/book.html +++ b/templates/book.html @@ -54,7 +54,7 @@ function delBook(){ {{end}}
- download + download read it!
diff --git a/trantor.go b/trantor.go index 1fcd09b..e48a93a 100644 --- a/trantor.go +++ b/trantor.go @@ -1,6 +1,7 @@ package main import ( + "io" "labix.org/v2/mgo/bson" "log" "net/http" @@ -65,9 +66,27 @@ func bookHandler(w http.ResponseWriter, r *http.Request) { } func downloadHandler(w http.ResponseWriter, r *http.Request) { - file := BOOKS_PATH + r.URL.Path[len("/books/"):] - db.IncDownload(file) - http.ServeFile(w, r, file) + id := bson.ObjectIdHex(r.URL.Path[len("/books/"):]) + books, _, err := db.GetBooks(bson.M{"_id": id}) + if err != nil || len(books) == 0 { + http.NotFound(w, r) + return + } + + fs := db.GetFS(FS_BOOKS) + f, err := fs.OpenId(books[0].File) + if err != nil { + http.NotFound(w, r) + return + } + defer f.Close() + + headers := w.Header() + headers["Content-Type"] = []string{"application/epub+zip"} + headers["Content-Disposition"] = []string{"attachment; filename=\"" + f.Name() + "\""} + + io.Copy(w, f) + db.IncDownload(id) } type indexData struct { @@ -97,10 +116,6 @@ func main() { /* create the needed folders */ var err error - _, err = os.Stat(BOOKS_PATH) - if err != nil { - os.Mkdir(BOOKS_PATH, os.ModePerm) - } _, err = os.Stat(COVER_PATH) if err != nil { os.Mkdir(COVER_PATH, os.ModePerm) diff --git a/upload.go b/upload.go index ba74729..f95f383 100644 --- a/upload.go +++ b/upload.go @@ -1,30 +1,30 @@ package main import ( + "labix.org/v2/mgo/bson" "log" "net/http" - "os" ) -func storeFiles(r *http.Request) ([]string, error) { +func storeFiles(r *http.Request) ([]bson.ObjectId, error) { r.ParseMultipartForm(20000000) filesForm := r.MultipartForm.File["epub"] - paths := make([]string, 0, len(filesForm)) + ids := make([]bson.ObjectId, 0, len(filesForm)) for _, f := range filesForm { log.Println("File uploaded:", f.Filename) file, err := f.Open() if err != nil { - return paths, err + return ids, err } defer file.Close() - path, err := StoreNewFile(f.Filename, file) + id, err := StoreNewFile(f.Filename, file) if err != nil { - return paths, err + return ids, err } - paths = append(paths, path) + ids = append(ids, id) } - return paths, nil + return ids, nil } type uploadData struct { @@ -34,17 +34,17 @@ type uploadData struct { func uploadHandler(w http.ResponseWriter, r *http.Request) { if r.Method == "POST" { sess := GetSession(r) - paths, err := storeFiles(r) + ids, err := storeFiles(r) if err != nil { sess.Notify("Problem uploading!", "Some files were not stored. Try again or contact us if it keeps happening", "error") } uploaded := "" - for _, path := range paths { - title, err := ParseFile(path) + for _, id := range ids { + title, err := ParseFile(id) if err != nil { - os.Remove(NEW_PATH + path) - sess.Notify("Problem uploading!", "The file '"+path+"' is not a well formed epub: "+err.Error(), "error") + DeleteFile(id) + sess.Notify("Problem uploading!", "The file is not a well formed epub: "+err.Error(), "error") } else { uploaded = uploaded + " '" + title + "'" }