From a1c029345bdea286a4de7f2cbcf00a1356b2998d Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Mon, 3 Jun 2013 10:21:28 +0200 Subject: [PATCH 01/13] Ignore ctags --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 865ec4a..9281d52 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,9 @@ books/ new/ cover/ trantor -.*.swp tools/adduser/adduser tools/update/update tools/togridfs/togridfs tools/getISBNnDesc/getISBNnDesc +tags +.*.swp From d51b7f935c06bcb661ec900879602cf825c5601f Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Tue, 4 Jun 2013 21:30:37 +0200 Subject: [PATCH 02/13] Use the new API of epubgo --- cover.go | 2 +- reader.go | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cover.go b/cover.go index 626f65f..1e8c68f 100644 --- a/cover.go +++ b/cover.go @@ -86,7 +86,7 @@ func GetCover(e *epubgo.Epub, title string) (bson.ObjectId, bson.ObjectId) { res := exp.FindSubmatch(txt) if res != nil { href := string(res[1]) - urlPart := strings.Split(it.Url(), "/") + urlPart := strings.Split(it.URL(), "/") url := strings.Join(urlPart[:len(urlPart)-1], "/") if href[:3] == "../" { href = href[3:] diff --git a/reader.go b/reader.go index 8b11a2b..4ab738d 100644 --- a/reader.go +++ b/reader.go @@ -65,10 +65,10 @@ func getNextPrev(e *epubgo.Epub, file string, id string, base string) (string, s prev := "" next := "" for err == nil { - if cleanLink(spine.Url()) == file { + if cleanLink(spine.URL()) == file { break } - prev = spine.Url() + prev = spine.URL() err = spine.Next() } if err != nil { @@ -79,7 +79,7 @@ func getNextPrev(e *epubgo.Epub, file string, id string, base string) (string, s prev = genLink(id, base, prev) } if spine.Next() == nil { - next = genLink(id, base, spine.Url()) + next = genLink(id, base, spine.URL()) } return next, prev } @@ -106,7 +106,7 @@ func listChapters(nav *epubgo.NavigationIterator, depth int) []chapter { for err == nil { var c chapter c.Label = nav.Title() - c.Link = nav.Url() + c.Link = nav.URL() c.Depth = depth for c.Depth < depth { c.Out = append(c.Out, true) @@ -143,7 +143,7 @@ func readStartHandler(w http.ResponseWriter, r *http.Request, sess *Session) { notFound(w, r) return } - http.Redirect(w, r, "/read/"+id+"/"+it.Url(), http.StatusTemporaryRedirect) + http.Redirect(w, r, "/read/"+id+"/"+it.URL(), http.StatusTemporaryRedirect) } func readHandler(w http.ResponseWriter, r *http.Request, sess *Session) { From 61764d51b329a67aafd86edc7a6dc7d0c0dc72cc Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Tue, 16 Jul 2013 01:24:39 +0200 Subject: [PATCH 03/13] Update about page with status, bitcoin and pgp --- templates/about.html | 69 ++++++++++++++++++++++++++++++++++++++++++++ templates/index.html | 5 ++++ 2 files changed, 74 insertions(+) diff --git a/templates/about.html b/templates/about.html index ecf1bf9..5475660 100644 --- a/templates/about.html +++ b/templates/about.html @@ -6,4 +6,73 @@

We like to pay the authors, but not the corporations that make profit from them. We won't listen to any content remove request from corporations, editorials, right management organizations or any other blood-suckers.

+

Status

+ +

The Imperial Library of Trantor it's in beta-status. We are working to provide a good user experience, but it's still in early development.

+ +

Any help is welcome. You can write us comments to our email address (zenow@tormail.org), upload your epubs, download our source code hack it and send us patches, ...

+ +

Donations

+ +

If you feel like donate some bitcoins we'll gladly accept them. You can request one bitcoin key for you or use our public bitcoin key:

+

1JioYbSYDH4JQYbhF7tX2kGUVZc2vzvugx

+ +

PGP

+ +

You can use pgp to write me emails, use my key:

+
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1.4.12 (GNU/Linux)
+
+mQINBFHkelgBEADG3Fh5MQTIlDI361//JoGtd+PiScLutyWAsuNICuWUwCSPSV9/
+tbYeptq6nawS/asu0Qzs6T99SqHypCq8WGbhBkHZaVPxBdnrfm7JLBfTeqCPMIm3
+KKeq2ipOg9bwUopEqYvpf85b7PnKRn1KpRj/Qz8majuihnFiLNHeRXnrYr89bWE8
+Xy45HPwngwvUFkZpE1ttBHGrDgb4VugnDiwRHyHG1MPl7k2YaZcKA5R+McaWuQZp
+MONs0/HFGR1PgRa/Kx5xCuGZe28PSgyQUF6FZbQjOR4IJoy5mqVy2OX43BggigHU
+/ISRQrYefs4jbBX6s2fT4lvgw4SSEQXPrSIw+E3xnPJk1OooniT6IKp9FtOGo2mb
+zCSiFm7Et97cpgo/FxANQr6DryEMeW+B68oZ0WrkeQZqKjXOloNwUUYizN+p7OsW
+K+k/VgRmqYerrXXqrdd73cuWjVx16c1SxiUoqV0LDoSVwlfpppS2WH3gg5xdjtAB
+7tsj3Q7OBejANYAvvE7XsWD+pg4Sy96htMAf+PVZipn03qoO9msb0MTHSzmjtNfJ
+bV19HjzKv3hzFG9qEh0g9Uc/Oyni8Q79ugwni54bYs38WjC8qQBTj0y3lkRE4JSP
+2zNaSubDvbSBOcncYh0m60b2NMIsXQoHHOMdLL1wMwLn91SBw1y25teV1QARAQAB
+tDtMYXMgWmVub3cgKEltcGVyaWFsIExpYnJhcnkgb2YgVHJhbnRvcikgPHplbm93
+QHRvcm1haWwub3JnPokCOAQTAQIAIgUCUeR6WAIbAwYLCQgHAwIGFQgCCQoLBBYC
+AwECHgECF4AACgkQRPcNrYeH/dewNA//dtATSYKwpt6itGQoZslqdaYbbxuhCWWz
+NA7VKATp1gQNhnnIJW8F/KziC39eKGGnSScS9La5Lazuj+CjV5SpXvlLFxfCr+vI
+G9PDEbgpCg1hmJTIAewVn3Xpx10F0fLmRw/DkeSdqECVeAT0WTIptHZh9M13fpa7
+urkewDaDLfQquASSkFUspAgUwJJfR0cQCO0d5XhRg2v9hz4pngrgwioDZKPa610v
+AuCSgrEgQF1D6YRfa3ePsWIHhKO1n1Kv2Qz9NNgdi5FQQOXKbttIU416VnHEnL6k
+RwtzRKE/K6KzCAtP4vwJaiJa4euGMFWO2kp608zCC11Ne6Bbszzt95QzKG9fKE80
+BwB4NFm+GZxJtk+nutF5XUvogl1+ogLyjCZxS4UNdnF2zb4hYvOoDv7GDD5I1/po
+MSp6ZetbIggWFGfMMh7Fe8HvqPLbV5H3hKQqx0zvzntROhs1a/EdfB1I0CfX75Ib
+KdxgBJ3ajFNtZc3z8m5IlOE2RAuKZ0zSVRl7cCX1D5tYsqRIGy4cvWJk5nKj5AMV
+101i/N26Fq1X/MOO5Cc2UKkLoZNv0PFMXwuoQx4iY4Y3KREz+08KYn4iM9u2d1Kq
+6icCWQ1aBQy84AQ9LQT5OwOBQltLKYaaucYciD28Fn32rDF1nfzH8m1zpNsRWTSq
+qj1kSZNJ5Xa5Ag0EUeR6WAEQALDfO5pnjJtVGAdqYaPbrf9fVnW5/RfYtLUrps8d
+UAduk6QFuCItb8OnuUDVqW08PEX5KcxA43wXLaO7Uvb3smBsWJOkOCwOgpEbTxnh
+98l6FnXLpMBkshh2jhsfWIKU3CH4qQNUYGc7MzXkMEeHZsJD5xvMauAnfarSFJLJ
+ij52D2Gm53kLcJjz/bn80qb6XhknFvQ3At5mEeUSRF73LOeTHc8pApIBn+aIUTXo
+8y5pPirr4k5aRc+cg06MNhjsvzyhzR7gT2r2QJjeuWGWA2Tl7xDudQwqZs69CMFd
+FZd9j/cOeTWwNtRZqstQ+jWBvP81T++MkTEsz9TOVZxSUcQ99qTQSGbH41KhlPDm
+mxD24xOA4RCMAKdHOELaU0wqb5DAHtJcJHEihczAaObzg8A3935vXob84UAqCcpG
+eIjegdueWkbgJECq+a0RFfiAbTkigDgvzDS45MkcCbSW9TSqzNhDFPGnrFDGlajq
+pCX8TZuzhOc5NupQo35h46z5pMCgNoPf2Jma771j2NJg73R/bjAo5wPdMlrMF0W5
+hckRTnm72Oeh87qXfwS7kgUSiXs2YLo869QAGbKw0IlMNCzDQ9t6/2IIKoCjcwLV
+c6tIYklUxz4N0S0ZQouZ0w84G+IyY2MyRKSRoky3fQLRIsNTBkWkVPl7IUhpXlPx
+9RYBABEBAAGJAh8EGAECAAkFAlHkelgCGwwACgkQRPcNrYeH/dd+QQ//WAvnIupJ
+psVkYn8CCld+3hUKohJhBQyKjQEgX04cH9jJlIBxkDqktFeUNQ/lwyG7Xu6wSSOv
+caGj+JsA9sJbvSv8ypMzxtbNEuPPvbUCQIprkkg9P46XTTUp3R5mWZ1bkxinODOd
+BcG8fezhwVG5D1x9eN9KtmoOyoXbZT6Lj1ngkLTBEu4qKXEmaBMrBtAebv1Jp+nl
+h63kH2e5JCgTAu1110Pp7oRwfv9b+LeKDyoR6BVgJbFUmDlZykg1JZkUCA3UkO5S
+xLDpTauc+dZ7iEBYE7MfHPNGYjk/1qd9us4a9ZdyFHC0FWctcPCkNBYtkEyF5I5S
+NUV3bJ5u3H+n9OTFCrZCWUuYjZ81ZK4PwvP9DWDirTG0pf2ilzi846NzN+cKQZVH
+KGW5abAAFVANWsNHaVfSdiH9zcsWru9li7Z4YalVkTBLDVg5xqJB0IB+9QkC2f8m
+eD5+TdHe3d9Ws+aTb55LY7QoV5yDK0k9VRn0jx5YU6rJPoeIz+ItRqyPNC6yGOb1
+mchroGN+i91cG0+sarVr26Xn4asasEOuPrmwH2YVBpzzN2bn1g3Z9pzb0Rmp0TpD
+Y5iwX9isk7J5yNkCl1bae0U5ztVtxB7ghXDUIVFalYvT6xn3p9t8iD3NsbML7IQ0
+IXsSLZzM+UxPlUZpEqSH0qllMrtZAR4Xvvk=
+=KPVB
+-----END PGP PUBLIC KEY BLOCK-----
+
+ {{template "footer.html"}} diff --git a/templates/index.html b/templates/index.html index e2fb587..ca8d66e 100644 --- a/templates/index.html +++ b/templates/index.html @@ -1,5 +1,10 @@ {{template "header.html" .S}} +
+ + News! The about has being updated with pgp key and bitcoin address for donations. +
+

Last books added:

From bf0480868dcfc0840a7cb5bf327e40965dbce7f3 Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Tue, 16 Jul 2013 22:56:48 +0200 Subject: [PATCH 04/13] Fix upload problems --- upload.go | 79 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 45 insertions(+), 34 deletions(-) diff --git a/upload.go b/upload.go index 2b90d76..74a6f43 100644 --- a/upload.go +++ b/upload.go @@ -18,51 +18,62 @@ func InitUpload() { var uploadChannel chan uploadRequest type uploadRequest struct { - epubs []*multipart.FileHeader + file multipart.File + filename string } func uploadWorker() { for req := range uploadChannel { - for _, f := range req.epubs { - file, err := f.Open() - if err != nil { - log.Println("Can not open uploaded file", f.Filename, ":", err) - continue - } - defer file.Close() - - epub, err := openMultipartEpub(file) - if err != nil { - log.Println("Not valid epub uploaded file", f.Filename, ":", err) - continue - } - defer epub.Close() - - book := parseFile(epub) - title, _ := book["title"].(string) - file.Seek(0, 0) - id, err := StoreNewFile(title+".epub", file) - if err != nil { - log.Println("Error storing book (", title, "):", err) - continue - } - - book["file"] = id - db.InsertBook(book) - log.Println("File uploaded:", f.Filename) - } + processFile(req) } } +func processFile(req uploadRequest) { + defer req.file.Close() + + epub, err := openMultipartEpub(req.file) + if err != nil { + log.Println("Not valid epub uploaded req.file", req.filename, ":", err) + return + } + defer epub.Close() + + book := parseFile(epub) + title, _ := book["title"].(string) + req.file.Seek(0, 0) + id, err := StoreNewFile(title+".epub", req.file) + if err != nil { + log.Println("Error storing book (", title, "):", err) + return + } + + book["req.file"] = id + db.InsertBook(book) + log.Println("File uploaded:", req.filename) +} + func uploadPostHandler(w http.ResponseWriter, r *http.Request, sess *Session) { + problem := false + r.ParseMultipartForm(20000000) filesForm := r.MultipartForm.File["epub"] - uploadChannel <- uploadRequest{filesForm} + for _, f := range filesForm { + file, err := f.Open() + if err != nil { + log.Println("Can not open uploaded file", f.Filename, ":", err) + sess.Notify("Upload problem!", "There was a problem with book "+f.Filename, "error") + problem = true + continue + } + uploadChannel <- uploadRequest{file, f.Filename} + } - if len(filesForm) > 0 { - sess.Notify("Upload successful!", "Thank you for your contribution", "success") - } else { - sess.Notify("Upload problem!", "No books where uploaded.", "error") + if !problem { + if len(filesForm) > 0 { + sess.Notify("Upload successful!", "Thank you for your contribution", "success") + } else { + sess.Notify("Upload problem!", "No books where uploaded.", "error") + } } uploadHandler(w, r, sess) } From 5f612a36adca598eb76c77e869aa5cd9f3b7d804 Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Wed, 17 Jul 2013 01:08:44 +0200 Subject: [PATCH 05/13] Simple news implementation Is missing some proper design and to show up the news on the front page. --- config.go | 2 ++ database.go | 19 +++++++++++++++ news.go | 51 ++++++++++++++++++++++++++++++++++++++++ template.go | 3 +++ templates/edit_news.html | 9 +++++++ templates/header.html | 5 +++- templates/news.html | 10 ++++++++ trantor.go | 3 +++ 8 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 news.go create mode 100644 templates/edit_news.html create mode 100644 templates/news.html diff --git a/config.go b/config.go index b8f5f71..21f0612 100644 --- a/config.go +++ b/config.go @@ -14,6 +14,7 @@ const ( DAILY_VISITS_COLL = "visits.daily" MONTHLY_VISITS_COLL = "visits.monthly" USERS_COLL = "users" + NEWS_COLL = "news" STATS_COLL = "statistics" FS_BOOKS = "fs_books" FS_IMGS = "fs_imgs" @@ -28,6 +29,7 @@ const ( TAGS_DISPLAY = 50 SEARCH_ITEMS_PAGE = 20 NEW_ITEMS_PAGE = 50 + NUM_NEWS = 10 TEMPLATE_PATH = "templates/" CSS_PATH = "css/" diff --git a/database.go b/database.go index ebf6fb0..9f96fd2 100644 --- a/database.go +++ b/database.go @@ -34,10 +34,16 @@ type Book struct { Keywords []string } +type News struct { + Date time.Time + Text string +} + type DB struct { session *mgo.Session books *mgo.Collection user *mgo.Collection + news *mgo.Collection stats *mgo.Collection mr *MR } @@ -53,6 +59,7 @@ func initDB() *DB { database := d.session.DB(DB_NAME) d.books = database.C(BOOKS_COLL) d.user = database.C(USERS_COLL) + d.news = database.C(NEWS_COLL) d.stats = database.C(STATS_COLL) d.mr = NewMR(database) return d @@ -82,6 +89,18 @@ func (d *DB) UserValid(user string, pass string) bool { return n != 0 } +func (d *DB) AddNews(text string) error { + var news News + news.Text = text + news.Date = time.Now() + return d.news.Insert(news) +} + +func (d *DB) GetNews(num int) (news []News, err error) { + err = d.news.Find(bson.M{}).Sort("-date").Limit(num).All(&news) + return +} + func (d *DB) InsertStats(stats interface{}) error { return d.stats.Insert(stats) } diff --git a/news.go b/news.go new file mode 100644 index 0000000..640c46d --- /dev/null +++ b/news.go @@ -0,0 +1,51 @@ +package main + +import ( + "net/http" +) + +type newsData struct { + S Status + News []news +} + +type news struct { + Date string + Text string +} + +func newsHandler(w http.ResponseWriter, r *http.Request, sess *Session) { + var data newsData + data.S = GetStatus(w, r) + data.S.News = true + newsEntries, _ := db.GetNews(NUM_NEWS) + data.News = make([]news, len(newsEntries)) + for i, n := range newsEntries { + data.News[i].Text = n.Text + data.News[i].Date = n.Date.Format("Jan 31, 2006") + } + loadTemplate(w, "news", data) +} + +func editNewsHandler(w http.ResponseWriter, r *http.Request, sess *Session) { + if sess.User == "" { + notFound(w, r) + return + } + + var data statusData + data.S = GetStatus(w, r) + data.S.News = true + loadTemplate(w, "edit_news", data) +} + +func postNewsHandler(w http.ResponseWriter, r *http.Request, sess *Session) { + if sess.User == "" { + notFound(w, r) + return + } + + text := r.FormValue("text") + db.AddNews(text) + http.Redirect(w, r, "/news/", http.StatusFound) +} diff --git a/template.go b/template.go index 6d5da0e..4d1ecaa 100644 --- a/template.go +++ b/template.go @@ -11,6 +11,7 @@ type Status struct { Notif []Notification Home bool About bool + News bool Upload bool Stats bool Help bool @@ -30,6 +31,8 @@ var templates = template.Must(template.ParseFiles(TEMPLATE_PATH+"header.html", TEMPLATE_PATH+"404.html", TEMPLATE_PATH+"index.html", TEMPLATE_PATH+"about.html", + TEMPLATE_PATH+"news.html", + TEMPLATE_PATH+"edit_news.html", TEMPLATE_PATH+"book.html", TEMPLATE_PATH+"search.html", TEMPLATE_PATH+"upload.html", diff --git a/templates/edit_news.html b/templates/edit_news.html new file mode 100644 index 0000000..d07c763 --- /dev/null +++ b/templates/edit_news.html @@ -0,0 +1,9 @@ +{{template "header.html" .S}} + +

Add News:

+
+ + +
+ +{{template "footer.html"}} diff --git a/templates/header.html b/templates/header.html index d46e17e..25d97c3 100644 --- a/templates/header.html +++ b/templates/header.html @@ -52,7 +52,8 @@ @@ -65,6 +66,8 @@
  • New books
    • +
  • Edit news
    • +
  • Settings
  • Log Out
    • diff --git a/templates/news.html b/templates/news.html new file mode 100644 index 0000000..45bcae8 --- /dev/null +++ b/templates/news.html @@ -0,0 +1,10 @@ +{{template "header.html" .S}} + +
    +{{range .News}} +
    {{.Date}}
    +
    {{.Text}}
    +{{end}} +
    + +{{template "footer.html"}} diff --git a/trantor.go b/trantor.go index ad2621d..b44adb5 100644 --- a/trantor.go +++ b/trantor.go @@ -181,6 +181,9 @@ func setUpRouter() { r.HandleFunc("/cover/{id:[0-9a-fA-F]+}/{size}/{img:.*}", coverHandler) r.HandleFunc("/settings/", GatherStats(settingsHandler)) r.HandleFunc("/stats/", GatherStats(statsHandler)) + r.HandleFunc("/news/", GatherStats(newsHandler)) + r.HandleFunc("/news/edit", GatherStats(editNewsHandler)).Methods("GET") + r.HandleFunc("/news/edit", GatherStats(postNewsHandler)).Methods("POST") h := http.FileServer(http.Dir(IMG_PATH)) r.Handle("/img/{img}", http.StripPrefix("/img/", h)) h = http.FileServer(http.Dir(CSS_PATH)) From 9a98cfd793c3cb34c93da3bd293b6f9afeac34ee Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Wed, 17 Jul 2013 15:33:09 +0200 Subject: [PATCH 06/13] Make the news look better --- templates/edit_news.html | 2 +- templates/news.html | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/templates/edit_news.html b/templates/edit_news.html index d07c763..1f0c5e6 100644 --- a/templates/edit_news.html +++ b/templates/edit_news.html @@ -2,7 +2,7 @@

    Add News:

    - +
    diff --git a/templates/news.html b/templates/news.html index 45bcae8..fe96ea3 100644 --- a/templates/news.html +++ b/templates/news.html @@ -1,9 +1,13 @@ {{template "header.html" .S}} +

    News:

    +
    {{range .News}} +
    {{.Date}}
    {{.Text}}
    +
    {{end}}
    From 54763b04df468da5be289f7053d3fcc6a29a6882 Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Thu, 18 Jul 2013 11:42:46 +0200 Subject: [PATCH 07/13] Print out the news in the front page --- config.go | 1 + database.go | 11 +++++++++-- news.go | 21 +++++++++++++-------- templates/index.html | 7 ++++--- trantor.go | 2 ++ 5 files changed, 29 insertions(+), 13 deletions(-) diff --git a/config.go b/config.go index 21f0612..f92f3a3 100644 --- a/config.go +++ b/config.go @@ -30,6 +30,7 @@ const ( SEARCH_ITEMS_PAGE = 20 NEW_ITEMS_PAGE = 50 NUM_NEWS = 10 + DAYS_NEWS_INDEXPAGE = 15 TEMPLATE_PATH = "templates/" CSS_PATH = "css/" diff --git a/database.go b/database.go index 9f96fd2..0dc0eec 100644 --- a/database.go +++ b/database.go @@ -96,8 +96,15 @@ func (d *DB) AddNews(text string) error { return d.news.Insert(news) } -func (d *DB) GetNews(num int) (news []News, err error) { - err = d.news.Find(bson.M{}).Sort("-date").Limit(num).All(&news) +func (d *DB) GetNews(num int, days int) (news []News, err error) { + query := bson.M{} + if days != 0 { + duration := time.Duration(-24*days) * time.Hour + date := time.Now().Add(duration) + query = bson.M{"date": bson.M{"$gt": date}} + } + q := d.news.Find(query).Sort("-date").Limit(num) + err = q.All(&news) return } diff --git a/news.go b/news.go index 640c46d..81ef543 100644 --- a/news.go +++ b/news.go @@ -6,10 +6,10 @@ import ( type newsData struct { S Status - News []news + News []newsEntry } -type news struct { +type newsEntry struct { Date string Text string } @@ -18,12 +18,7 @@ func newsHandler(w http.ResponseWriter, r *http.Request, sess *Session) { var data newsData data.S = GetStatus(w, r) data.S.News = true - newsEntries, _ := db.GetNews(NUM_NEWS) - data.News = make([]news, len(newsEntries)) - for i, n := range newsEntries { - data.News[i].Text = n.Text - data.News[i].Date = n.Date.Format("Jan 31, 2006") - } + data.News = getNews(NUM_NEWS, 0) loadTemplate(w, "news", data) } @@ -49,3 +44,13 @@ func postNewsHandler(w http.ResponseWriter, r *http.Request, sess *Session) { db.AddNews(text) http.Redirect(w, r, "/news/", http.StatusFound) } + +func getNews(num int, days int) []newsEntry { + dbnews, _ := db.GetNews(num, days) + news := make([]newsEntry, len(dbnews)) + for i, n := range dbnews { + news[i].Text = n.Text + news[i].Date = n.Date.Format("Jan 31, 2006") + } + return news +} diff --git a/templates/index.html b/templates/index.html index ca8d66e..cd0f7a5 100644 --- a/templates/index.html +++ b/templates/index.html @@ -1,9 +1,10 @@ {{template "header.html" .S}} -
    - - News! The about has being updated with pgp key and bitcoin address for donations. +{{range .News}} +
    + News! {{.Text}}
    +{{end}}
    diff --git a/trantor.go b/trantor.go index b44adb5..3c284f0 100644 --- a/trantor.go +++ b/trantor.go @@ -121,6 +121,7 @@ type indexData struct { DownloadedBooks []Book Count int Tags []string + News []newsEntry } func indexHandler(w http.ResponseWriter, r *http.Request, sess *Session) { @@ -132,6 +133,7 @@ func indexHandler(w http.ResponseWriter, r *http.Request, sess *Session) { data.Books, data.Count, _ = db.GetBooks(bson.M{"active": true}, 6) data.VisitedBooks, _ = db.GetVisitedBooks(6) data.DownloadedBooks, _ = db.GetDownloadedBooks(6) + data.News = getNews(1, DAYS_NEWS_INDEXPAGE) loadTemplate(w, "index", data) } From d1d99704275689ee16c82d59e77eac5e69982e20 Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Thu, 18 Jul 2013 11:54:51 +0200 Subject: [PATCH 08/13] Fix the date format --- news.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news.go b/news.go index 81ef543..cae1f56 100644 --- a/news.go +++ b/news.go @@ -50,7 +50,7 @@ func getNews(num int, days int) []newsEntry { news := make([]newsEntry, len(dbnews)) for i, n := range dbnews { news[i].Text = n.Text - news[i].Date = n.Date.Format("Jan 31, 2006") + news[i].Date = n.Date.Format("Jan 2, 2006") } return news } From 6130ffe7b7b5c47fa73efeb88918ac3e21feb7d9 Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Thu, 18 Jul 2013 20:28:35 +0200 Subject: [PATCH 09/13] Detect Date: Thu, 18 Jul 2013 21:08:22 +0200 Subject: [PATCH 10/13] Extract the cover from the metadata --- cover.go | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/cover.go b/cover.go index 85004df..7444567 100644 --- a/cover.go +++ b/cover.go @@ -64,7 +64,12 @@ func coverHandler(w http.ResponseWriter, r *http.Request) { } func GetCover(e *epubgo.Epub, title string) (bson.ObjectId, bson.ObjectId) { - imgId, smallId := searchCommonCoverNames(e, title) + imgId, smallId := coverFromMetadata(e, title) + if imgId != "" { + return imgId, smallId + } + + imgId, smallId = searchCommonCoverNames(e, title) if imgId != "" { return imgId, smallId } @@ -113,8 +118,22 @@ func GetCover(e *epubgo.Epub, title string) (bson.ObjectId, bson.ObjectId) { return "", "" } +func coverFromMetadata(e *epubgo.Epub, title string) (bson.ObjectId, bson.ObjectId) { + metaList, _ := e.MetadataAttr("meta") + for _, meta := range metaList { + if meta["name"] == "cover" { + img, err := e.OpenFileId(meta["content"]) + if err == nil { + defer img.Close() + return storeImg(img, title) + } + } + } + return "", "" +} + func searchCommonCoverNames(e *epubgo.Epub, title string) (bson.ObjectId, bson.ObjectId) { - for _, p := range []string{"cover.jpg", "Images/cover.jpg", "cover.jpeg", "cover1.jpg", "cover1.jpeg"} { + for _, p := range []string{"cover.jpg", "Images/cover.jpg", "images/cover.jpg", "cover.jpeg", "cover1.jpg", "cover1.jpeg"} { img, err := e.OpenFile(p) if err == nil { defer img.Close() From c01c465330d194b820b6de90193ff4d5d6d170cd Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Thu, 18 Jul 2013 21:53:35 +0200 Subject: [PATCH 11/13] Store properly the book on the database --- upload.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/upload.go b/upload.go index 74a6f43..f58c526 100644 --- a/upload.go +++ b/upload.go @@ -33,7 +33,7 @@ func processFile(req uploadRequest) { epub, err := openMultipartEpub(req.file) if err != nil { - log.Println("Not valid epub uploaded req.file", req.filename, ":", err) + log.Println("Not valid epub uploaded file", req.filename, ":", err) return } defer epub.Close() @@ -47,7 +47,7 @@ func processFile(req uploadRequest) { return } - book["req.file"] = id + book["file"] = id db.InsertBook(book) log.Println("File uploaded:", req.filename) } From 4b536352de7bfccbeb9092b54f372a7f19aa3e62 Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Thu, 18 Jul 2013 22:08:35 +0200 Subject: [PATCH 12/13] Dirty hack to reload covers from new books --- .gitignore | 1 + tools/README | 4 +- tools/coverNew/config.go | 45 ++++++ tools/coverNew/cover.go | 204 +++++++++++++++++++++++++++ tools/coverNew/coverNew.go | 32 +++++ tools/coverNew/database.go | 230 +++++++++++++++++++++++++++++++ tools/coverNew/mapreduce.go | 266 ++++++++++++++++++++++++++++++++++++ tools/coverNew/session.go | 76 +++++++++++ tools/coverNew/store.go | 128 +++++++++++++++++ 9 files changed, 985 insertions(+), 1 deletion(-) create mode 100644 tools/coverNew/config.go create mode 100644 tools/coverNew/cover.go create mode 100644 tools/coverNew/coverNew.go create mode 100644 tools/coverNew/database.go create mode 100644 tools/coverNew/mapreduce.go create mode 100644 tools/coverNew/session.go create mode 100644 tools/coverNew/store.go diff --git a/.gitignore b/.gitignore index 9281d52..2392860 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,6 @@ tools/adduser/adduser tools/update/update tools/togridfs/togridfs tools/getISBNnDesc/getISBNnDesc +tools/coverNew/coverNew tags .*.swp diff --git a/tools/README b/tools/README index cadb0d8..ece1903 100644 --- a/tools/README +++ b/tools/README @@ -1,4 +1,4 @@ -Some tools dirty to manage trantor: +Some dirty tools to manage trantor: - adduser. Used to add users to trantor: $ adduser myNick @@ -9,3 +9,5 @@ Password: - togridfs (23/4/2013). Migrate all files and covers to gridfs - getISBNnDesc (31/5/2013). Import the ISBN and the description with changes of lines to the database + +- coverNew. Reload the cover from all the new books diff --git a/tools/coverNew/config.go b/tools/coverNew/config.go new file mode 100644 index 0000000..f92f3a3 --- /dev/null +++ b/tools/coverNew/config.go @@ -0,0 +1,45 @@ +package main + +const ( + PORT = "8080" + + DB_IP = "127.0.0.1" + DB_NAME = "trantor" + META_COLL = "meta" + BOOKS_COLL = "books" + TAGS_COLL = "tags" + VISITED_COLL = "visited" + DOWNLOADED_COLL = "downloaded" + HOURLY_VISITS_COLL = "visits.hourly" + DAILY_VISITS_COLL = "visits.daily" + MONTHLY_VISITS_COLL = "visits.monthly" + USERS_COLL = "users" + NEWS_COLL = "news" + STATS_COLL = "statistics" + FS_BOOKS = "fs_books" + FS_IMGS = "fs_imgs" + + PASS_SALT = "ImperialLibSalt" + MINUTES_UPDATE_TAGS = 11 + MINUTES_UPDATE_VISITED = 41 + MINUTES_UPDATE_DOWNLOADED = 47 + MINUTES_UPDATE_HOURLY = 31 + MINUTES_UPDATE_DAILY = 60*12 + 7 + MINUTES_UPDATE_MONTHLY = 60*24 + 11 + TAGS_DISPLAY = 50 + SEARCH_ITEMS_PAGE = 20 + NEW_ITEMS_PAGE = 50 + NUM_NEWS = 10 + DAYS_NEWS_INDEXPAGE = 15 + + TEMPLATE_PATH = "templates/" + CSS_PATH = "css/" + JS_PATH = "js/" + IMG_PATH = "img/" + + IMG_WIDTH_BIG = 300 + IMG_WIDTH_SMALL = 60 + IMG_QUALITY = 80 + + CHAN_SIZE = 100 +) diff --git a/tools/coverNew/cover.go b/tools/coverNew/cover.go new file mode 100644 index 0000000..7444567 --- /dev/null +++ b/tools/coverNew/cover.go @@ -0,0 +1,204 @@ +package main + +import _ "image/png" +import _ "image/jpeg" +import _ "image/gif" + +import ( + "bytes" + "git.gitorious.org/go-pkg/epubgo.git" + "github.com/gorilla/mux" + "github.com/nfnt/resize" + "image" + "image/jpeg" + "io" + "io/ioutil" + "labix.org/v2/mgo" + "labix.org/v2/mgo/bson" + "log" + "net/http" + "regexp" + "strings" +) + +func coverHandler(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + if !bson.IsObjectIdHex(vars["id"]) { + notFound(w, r) + return + } + id := bson.ObjectIdHex(vars["id"]) + books, _, err := db.GetBooks(bson.M{"_id": id}) + if err != nil || len(books) == 0 { + notFound(w, r) + return + } + book := books[0] + + if !book.Active { + sess := GetSession(r) + if sess.User == "" { + notFound(w, r) + return + } + } + + fs := db.GetFS(FS_IMGS) + var f *mgo.GridFile + if vars["size"] == "small" { + f, err = fs.OpenId(book.CoverSmall) + } else { + f, err = fs.OpenId(book.Cover) + } + if err != nil { + log.Println("Error while opening image:", err) + notFound(w, r) + return + } + defer f.Close() + + headers := w.Header() + headers["Content-Type"] = []string{"image/jpeg"} + + io.Copy(w, f) +} + +func GetCover(e *epubgo.Epub, title string) (bson.ObjectId, bson.ObjectId) { + imgId, smallId := coverFromMetadata(e, title) + if imgId != "" { + return imgId, smallId + } + + imgId, smallId = searchCommonCoverNames(e, title) + if imgId != "" { + return imgId, smallId + } + + /* search for img on the text */ + exp, _ := regexp.Compile("<.*ima?g.*[(src)(href)]=[\"']([^\"']*(\\.[^\\.\"']*))[\"']") + it, errNext := e.Spine() + for errNext == nil { + file, err := it.Open() + if err != nil { + break + } + defer file.Close() + + txt, err := ioutil.ReadAll(file) + if err != nil { + break + } + res := exp.FindSubmatch(txt) + if res != nil { + href := string(res[1]) + urlPart := strings.Split(it.URL(), "/") + url := strings.Join(urlPart[:len(urlPart)-1], "/") + if href[:3] == "../" { + href = href[3:] + url = strings.Join(urlPart[:len(urlPart)-2], "/") + } + href = strings.Replace(href, "%20", " ", -1) + href = strings.Replace(href, "%27", "'", -1) + href = strings.Replace(href, "%28", "(", -1) + href = strings.Replace(href, "%29", ")", -1) + if url == "" { + url = href + } else { + url = url + "/" + href + } + + img, err := e.OpenFile(url) + if err == nil { + defer img.Close() + return storeImg(img, title) + } + } + errNext = it.Next() + } + return "", "" +} + +func coverFromMetadata(e *epubgo.Epub, title string) (bson.ObjectId, bson.ObjectId) { + metaList, _ := e.MetadataAttr("meta") + for _, meta := range metaList { + if meta["name"] == "cover" { + img, err := e.OpenFileId(meta["content"]) + if err == nil { + defer img.Close() + return storeImg(img, title) + } + } + } + return "", "" +} + +func searchCommonCoverNames(e *epubgo.Epub, title string) (bson.ObjectId, bson.ObjectId) { + for _, p := range []string{"cover.jpg", "Images/cover.jpg", "images/cover.jpg", "cover.jpeg", "cover1.jpg", "cover1.jpeg"} { + img, err := e.OpenFile(p) + if err == nil { + defer img.Close() + return storeImg(img, title) + } + } + return "", "" +} + +func storeImg(img io.Reader, title string) (bson.ObjectId, bson.ObjectId) { + /* open the files */ + fBig, err := createCoverFile(title) + if err != nil { + log.Println("Error creating", title, ":", err.Error()) + return "", "" + } + defer fBig.Close() + + fSmall, err := createCoverFile(title + "_small") + if err != nil { + log.Println("Error creating", title+"_small", ":", err.Error()) + return "", "" + } + defer fSmall.Close() + + /* resize img */ + var img2 bytes.Buffer + img1 := io.TeeReader(img, &img2) + jpgOptions := jpeg.Options{IMG_QUALITY} + imgResized, err := resizeImg(img1, IMG_WIDTH_BIG) + if err != nil { + log.Println("Error resizing big image:", err.Error()) + return "", "" + } + err = jpeg.Encode(fBig, imgResized, &jpgOptions) + if err != nil { + log.Println("Error encoding big image:", err.Error()) + return "", "" + } + imgSmallResized, err := resizeImg(&img2, IMG_WIDTH_SMALL) + if err != nil { + log.Println("Error resizing small image:", err.Error()) + return "", "" + } + err = jpeg.Encode(fSmall, imgSmallResized, &jpgOptions) + if err != nil { + log.Println("Error encoding small image:", err.Error()) + return "", "" + } + + idBig, _ := fBig.Id().(bson.ObjectId) + idSmall, _ := fSmall.Id().(bson.ObjectId) + return idBig, idSmall +} + +func createCoverFile(title string) (*mgo.GridFile, error) { + fs := db.GetFS(FS_IMGS) + return fs.Create(title + ".jpg") +} + +func resizeImg(imgReader io.Reader, width uint) (image.Image, error) { + img, _, err := image.Decode(imgReader) + if err != nil { + return nil, err + } + + return resize.Resize(width, 0, img, resize.NearestNeighbor), nil +} diff --git a/tools/coverNew/coverNew.go b/tools/coverNew/coverNew.go new file mode 100644 index 0000000..b283157 --- /dev/null +++ b/tools/coverNew/coverNew.go @@ -0,0 +1,32 @@ +package main + +import ( + "fmt" + "labix.org/v2/mgo/bson" + "net/http" +) + +func main() { + db = initDB() + defer db.Close() + books, _, _ := db.GetNewBooks() + + for _, book := range books { + fmt.Println(book.Title) + fmt.Println(book.File) + e, err := OpenBook(book.File) + if err != nil { + fmt.Println("================", err) + } + + cover, coverSmall := GetCover(e, book.Title) + if cover != "" { + db.UpdateBook(bson.ObjectIdHex(book.Id), bson.M{"cover": cover, "coversmall": coverSmall}) + } + e.Close() + } +} + +func notFound(w http.ResponseWriter, r *http.Request) { + // cover.go needs this function to compile +} diff --git a/tools/coverNew/database.go b/tools/coverNew/database.go new file mode 100644 index 0000000..0dc0eec --- /dev/null +++ b/tools/coverNew/database.go @@ -0,0 +1,230 @@ +package main + +import ( + "crypto/md5" + "labix.org/v2/mgo" + "labix.org/v2/mgo/bson" + "time" +) + +var db *DB + +type Book struct { + Id string `bson:"_id"` + Title string + Author []string + Contributor string + Publisher string + Description string + Subject []string + Date string + Lang []string + Isbn string + Type string + Format string + Source string + Relation string + Coverage string + Rights string + Meta string + File bson.ObjectId + Cover bson.ObjectId + CoverSmall bson.ObjectId + Active bool + Keywords []string +} + +type News struct { + Date time.Time + Text string +} + +type DB struct { + session *mgo.Session + books *mgo.Collection + user *mgo.Collection + news *mgo.Collection + stats *mgo.Collection + mr *MR +} + +func initDB() *DB { + var err error + d := new(DB) + d.session, err = mgo.Dial(DB_IP) + if err != nil { + panic(err) + } + + database := d.session.DB(DB_NAME) + d.books = database.C(BOOKS_COLL) + d.user = database.C(USERS_COLL) + d.news = database.C(NEWS_COLL) + d.stats = database.C(STATS_COLL) + d.mr = NewMR(database) + return d +} + +func (d *DB) Close() { + d.session.Close() +} + +func md5Pass(pass string) []byte { + h := md5.New() + hash := h.Sum(([]byte)(PASS_SALT + pass)) + return hash +} + +func (d *DB) SetPassword(user string, pass string) error { + hash := md5Pass(pass) + return d.user.Update(bson.M{"user": user}, bson.M{"$set": bson.M{"pass": hash}}) +} + +func (d *DB) UserValid(user string, pass string) bool { + hash := md5Pass(pass) + n, err := d.user.Find(bson.M{"user": user, "pass": hash}).Count() + if err != nil { + return false + } + return n != 0 +} + +func (d *DB) AddNews(text string) error { + var news News + news.Text = text + news.Date = time.Now() + return d.news.Insert(news) +} + +func (d *DB) GetNews(num int, days int) (news []News, err error) { + query := bson.M{} + if days != 0 { + duration := time.Duration(-24*days) * time.Hour + date := time.Now().Add(duration) + query = bson.M{"date": bson.M{"$gt": date}} + } + q := d.news.Find(query).Sort("-date").Limit(num) + err = q.All(&news) + return +} + +func (d *DB) InsertStats(stats interface{}) error { + return d.stats.Insert(stats) +} + +func (d *DB) InsertBook(book interface{}) error { + return d.books.Insert(book) +} + +func (d *DB) RemoveBook(id bson.ObjectId) error { + return d.books.Remove(bson.M{"_id": id}) +} + +func (d *DB) UpdateBook(id bson.ObjectId, data interface{}) error { + return d.books.Update(bson.M{"_id": id}, bson.M{"$set": data}) +} + +/* optional parameters: length and start index + * + * Returns: list of books, number found and err + */ +func (d *DB) GetBooks(query bson.M, r ...int) (books []Book, num int, err error) { + var start, length int + if len(r) > 0 { + length = r[0] + if len(r) > 1 { + start = r[1] + } + } + q := d.books.Find(query).Sort("-_id") + num, err = q.Count() + if err != nil { + return + } + if start != 0 { + q = q.Skip(start) + } + if length != 0 { + q = q.Limit(length) + } + + err = q.All(&books) + for i, b := range books { + books[i].Id = bson.ObjectId(b.Id).Hex() + } + return +} + +/* Get the most visited books + */ +func (d *DB) GetVisitedBooks(num int) (books []Book, err error) { + bookId, err := d.mr.GetMostVisited(num, d.stats) + if err != nil { + return nil, err + } + + books = make([]Book, num) + for i, id := range bookId { + d.books.Find(bson.M{"_id": id}).One(&books[i]) + books[i].Id = bson.ObjectId(books[i].Id).Hex() + } + return +} + +/* Get the most downloaded books + */ +func (d *DB) GetDownloadedBooks(num int) (books []Book, err error) { + bookId, err := d.mr.GetMostDownloaded(num, d.stats) + if err != nil { + return nil, err + } + + books = make([]Book, num) + for i, id := range bookId { + d.books.Find(bson.M{"_id": id}).One(&books[i]) + books[i].Id = bson.ObjectId(books[i].Id).Hex() + } + return +} + +/* optional parameters: length and start index + * + * Returns: list of books, number found and err + */ +func (d *DB) GetNewBooks(r ...int) (books []Book, num int, err error) { + return d.GetBooks(bson.M{"$nor": []bson.M{{"active": true}}}, r...) +} + +func (d *DB) BookActive(id bson.ObjectId) bool { + var book Book + err := d.books.Find(bson.M{"_id": id}).One(&book) + if err != nil { + return false + } + return book.Active +} + +func (d *DB) GetFS(prefix string) *mgo.GridFS { + return d.session.DB(DB_NAME).GridFS(prefix) +} + +func (d *DB) GetTags(numTags int) ([]string, error) { + return d.mr.GetTags(numTags, d.books) +} + +type Visits struct { + Date int64 "_id" + Count int "value" +} + +func (d *DB) GetHourVisits(start time.Time) ([]Visits, error) { + return d.mr.GetHourVisits(start, d.stats) +} + +func (d *DB) GetDayVisits(start time.Time) ([]Visits, error) { + return d.mr.GetDayVisits(start, d.stats) +} + +func (d *DB) GetMonthVisits(start time.Time) ([]Visits, error) { + return d.mr.GetMonthVisits(start, d.stats) +} diff --git a/tools/coverNew/mapreduce.go b/tools/coverNew/mapreduce.go new file mode 100644 index 0000000..dbadd19 --- /dev/null +++ b/tools/coverNew/mapreduce.go @@ -0,0 +1,266 @@ +package main + +import ( + "labix.org/v2/mgo" + "labix.org/v2/mgo/bson" + "time" +) + +type MR struct { + meta *mgo.Collection + tags *mgo.Collection + visited *mgo.Collection + downloaded *mgo.Collection + hourly_raw *mgo.Collection + daily_raw *mgo.Collection + monthly_raw *mgo.Collection + hourly *mgo.Collection + daily *mgo.Collection + monthly *mgo.Collection +} + +func NewMR(database *mgo.Database) *MR { + m := new(MR) + m.meta = database.C(META_COLL) + m.tags = database.C(TAGS_COLL) + m.visited = database.C(VISITED_COLL) + m.downloaded = database.C(DOWNLOADED_COLL) + m.hourly_raw = database.C(HOURLY_VISITS_COLL + "_raw") + m.daily_raw = database.C(DAILY_VISITS_COLL + "_raw") + m.monthly_raw = database.C(MONTHLY_VISITS_COLL + "_raw") + m.hourly = database.C(HOURLY_VISITS_COLL) + m.daily = database.C(DAILY_VISITS_COLL) + m.monthly = database.C(MONTHLY_VISITS_COLL) + return m +} + +func (m *MR) GetTags(numTags int, booksColl *mgo.Collection) ([]string, error) { + if m.isOutdated(TAGS_COLL, MINUTES_UPDATE_TAGS) { + var mr mgo.MapReduce + mr.Map = `function() { + if (this.subject) { + this.subject.forEach(function(s) { emit(s, 1); }); + } + }` + mr.Reduce = `function(tag, vals) { + var count = 0; + vals.forEach(function() { count += 1; }); + return count; + }` + err := m.update(&mr, bson.M{"active": true}, booksColl, TAGS_COLL) + if err != nil { + return nil, err + } + } + + var result []struct { + Tag string "_id" + } + err := m.tags.Find(nil).Sort("-value").Limit(numTags).All(&result) + if err != nil { + return nil, err + } + + tags := make([]string, len(result)) + for i, r := range result { + tags[i] = r.Tag + } + return tags, nil +} + +func (m *MR) GetMostVisited(num int, statsColl *mgo.Collection) ([]bson.ObjectId, error) { + if m.isOutdated(VISITED_COLL, MINUTES_UPDATE_VISITED) { + var mr mgo.MapReduce + mr.Map = `function() { + emit(this.id, 1); + }` + mr.Reduce = `function(tag, vals) { + var count = 0; + vals.forEach(function() { count += 1; }); + return count; + }` + err := m.update(&mr, bson.M{"section": "book"}, statsColl, VISITED_COLL) + if err != nil { + return nil, err + } + } + + var result []struct { + Book bson.ObjectId "_id" + } + err := m.visited.Find(nil).Sort("-value").Limit(num).All(&result) + if err != nil { + return nil, err + } + + books := make([]bson.ObjectId, len(result)) + for i, r := range result { + books[i] = r.Book + } + return books, nil +} + +func (m *MR) GetMostDownloaded(num int, statsColl *mgo.Collection) ([]bson.ObjectId, error) { + if m.isOutdated(DOWNLOADED_COLL, MINUTES_UPDATE_DOWNLOADED) { + var mr mgo.MapReduce + mr.Map = `function() { + emit(this.id, 1); + }` + mr.Reduce = `function(tag, vals) { + var count = 0; + vals.forEach(function() { count += 1; }); + return count; + }` + err := m.update(&mr, bson.M{"section": "download"}, statsColl, DOWNLOADED_COLL) + if err != nil { + return nil, err + } + } + + var result []struct { + Book bson.ObjectId "_id" + } + err := m.downloaded.Find(nil).Sort("-value").Limit(num).All(&result) + if err != nil { + return nil, err + } + + books := make([]bson.ObjectId, len(result)) + for i, r := range result { + books[i] = r.Book + } + return books, nil +} + +func (m *MR) GetHourVisits(start time.Time, statsColl *mgo.Collection) ([]Visits, error) { + if m.isOutdated(HOURLY_VISITS_COLL, MINUTES_UPDATE_HOURLY) { + const reduce = `function(date, vals) { + var count = 0; + vals.forEach(function(v) { count += v; }); + return count; + }` + var mr mgo.MapReduce + mr.Map = `function() { + var date = Date.UTC(this.date.getUTCFullYear(), + this.date.getUTCMonth(), + this.date.getUTCDate(), + this.date.getUTCHours()); + emit({date: date, session: this.session}, 1); + }` + mr.Reduce = reduce + err := m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, HOURLY_VISITS_COLL+"_raw") + if err != nil { + return nil, err + } + var mr2 mgo.MapReduce + mr2.Map = `function() { + emit(this['_id']['date'], 1); + }` + mr2.Reduce = reduce + err = m.update(&mr2, bson.M{}, m.hourly_raw, HOURLY_VISITS_COLL) + if err != nil { + return nil, err + } + } + + var result []Visits + err := m.hourly.Find(nil).All(&result) + return result, err +} + +func (m *MR) GetDayVisits(start time.Time, statsColl *mgo.Collection) ([]Visits, error) { + if m.isOutdated(DAILY_VISITS_COLL, MINUTES_UPDATE_DAILY) { + const reduce = `function(date, vals) { + var count = 0; + vals.forEach(function(v) { count += v; }); + return count; + }` + var mr mgo.MapReduce + mr.Map = `function() { + var date = Date.UTC(this.date.getUTCFullYear(), + this.date.getUTCMonth(), + this.date.getUTCDate()); + emit({date: date, session: this.session}, 1); + }` + mr.Reduce = reduce + err := m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, DAILY_VISITS_COLL+"_raw") + if err != nil { + return nil, err + } + var mr2 mgo.MapReduce + mr2.Map = `function() { + emit(this['_id']['date'], 1); + }` + mr2.Reduce = reduce + err = m.update(&mr2, bson.M{}, m.daily_raw, DAILY_VISITS_COLL) + if err != nil { + return nil, err + } + } + + var result []Visits + err := m.daily.Find(nil).All(&result) + return result, err +} + +func (m *MR) GetMonthVisits(start time.Time, statsColl *mgo.Collection) ([]Visits, error) { + if m.isOutdated(MONTHLY_VISITS_COLL, MINUTES_UPDATE_MONTHLY) { + const reduce = `function(date, vals) { + var count = 0; + vals.forEach(function(v) { count += v; }); + return count; + }` + var mr mgo.MapReduce + mr.Map = `function() { + var date = Date.UTC(this.date.getUTCFullYear(), + this.date.getUTCMonth()); + emit({date: date, session: this.session}, 1); + }` + mr.Reduce = reduce + err := m.update(&mr, bson.M{"date": bson.M{"$gte": start}}, statsColl, MONTHLY_VISITS_COLL+"_raw") + if err != nil { + return nil, err + } + var mr2 mgo.MapReduce + mr2.Map = `function() { + emit(this['_id']['date'], 1); + }` + mr2.Reduce = reduce + err = m.update(&mr2, bson.M{}, m.monthly_raw, MONTHLY_VISITS_COLL) + if err != nil { + return nil, err + } + } + + var result []Visits + err := m.monthly.Find(nil).All(&result) + return result, err +} + +func (m *MR) update(mr *mgo.MapReduce, query bson.M, queryColl *mgo.Collection, storeColl string) error { + _, err := m.meta.RemoveAll(bson.M{"type": storeColl}) + if err != nil { + return err + } + + mr.Out = bson.M{"replace": storeColl} + _, err = queryColl.Find(query).MapReduce(mr, nil) + if err != nil { + return err + } + + return m.meta.Insert(bson.M{"type": storeColl}) +} + +func (m *MR) isOutdated(coll string, minutes float64) bool { + var result struct { + Id bson.ObjectId `bson:"_id"` + } + err := m.meta.Find(bson.M{"type": coll}).One(&result) + if err != nil { + return true + } + + lastUpdate := result.Id.Time() + return time.Since(lastUpdate).Minutes() > minutes +} diff --git a/tools/coverNew/session.go b/tools/coverNew/session.go new file mode 100644 index 0000000..bf861fc --- /dev/null +++ b/tools/coverNew/session.go @@ -0,0 +1,76 @@ +package main + +import ( + "encoding/hex" + "github.com/gorilla/securecookie" + "github.com/gorilla/sessions" + "net/http" +) + +var sesStore = sessions.NewCookieStore(securecookie.GenerateRandomKey(64)) + +type Notification struct { + Title string + Msg string + Type string /* error, info or success */ +} + +type Session struct { + User string + Notif []Notification + S *sessions.Session +} + +func getNotif(session *sessions.Session) []Notification { + msgs := session.Flashes("nMsg") + titles := session.Flashes("nTitle") + tpes := session.Flashes("nType") + notif := make([]Notification, len(msgs)) + for i, m := range msgs { + msg, _ := m.(string) + title, _ := titles[i].(string) + tpe, _ := tpes[i].(string) + notif[i] = Notification{title, msg, tpe} + } + return notif +} + +func GetSession(r *http.Request) (s *Session) { + s = new(Session) + var err error + s.S, err = sesStore.Get(r, "session") + if err == nil && !s.S.IsNew { + s.User, _ = s.S.Values["user"].(string) + s.Notif = getNotif(s.S) + } + + if s.S.IsNew { + s.S.Values["id"] = hex.EncodeToString(securecookie.GenerateRandomKey(16)) + } + + return +} + +func (s *Session) LogIn(user string) { + s.User = user + s.S.Values["user"] = user +} + +func (s *Session) LogOut() { + s.S.Values["user"] = "" +} + +func (s *Session) Notify(title, msg, tpe string) { + s.S.AddFlash(msg, "nMsg") + s.S.AddFlash(title, "nTitle") + s.S.AddFlash(tpe, "nType") +} + +func (s *Session) Save(w http.ResponseWriter, r *http.Request) { + sesStore.Save(r, w, s.S) +} + +func (s *Session) Id() string { + id, _ := s.S.Values["id"].(string) + return id +} diff --git a/tools/coverNew/store.go b/tools/coverNew/store.go new file mode 100644 index 0000000..e50e65f --- /dev/null +++ b/tools/coverNew/store.go @@ -0,0 +1,128 @@ +package main + +import ( + "bytes" + "git.gitorious.org/go-pkg/epubgo.git" + "io" + "io/ioutil" + "labix.org/v2/mgo/bson" + "regexp" + "strings" +) + +func OpenBook(id bson.ObjectId) (*epubgo.Epub, error) { + fs := db.GetFS(FS_BOOKS) + f, err := fs.OpenId(id) + if err != nil { + return nil, err + } + defer f.Close() + + buff, err := ioutil.ReadAll(f) + reader := bytes.NewReader(buff) + + return epubgo.Load(reader, int64(len(buff))) +} + +func StoreNewFile(name string, file io.Reader) (bson.ObjectId, error) { + fs := db.GetFS(FS_BOOKS) + fw, err := fs.Create(name) + if err != nil { + return "", err + } + defer fw.Close() + + _, err = io.Copy(fw, file) + id, _ := fw.Id().(bson.ObjectId) + return id, err +} + +func DeleteFile(id bson.ObjectId) error { + fs := db.GetFS(FS_BOOKS) + return fs.RemoveId(id) +} + +func DeleteCover(id bson.ObjectId) error { + fs := db.GetFS(FS_IMGS) + return fs.RemoveId(id) +} + +func DeleteBook(book Book) { + if book.Cover != "" { + DeleteCover(book.Cover) + } + if book.CoverSmall != "" { + DeleteCover(book.CoverSmall) + } + DeleteFile(book.File) +} + +func cleanStr(str string) string { + str = strings.Replace(str, "'", "'", -1) + exp, _ := regexp.Compile("&[^;]*;") + str = exp.ReplaceAllString(str, "") + exp, _ = regexp.Compile("[ ,]*$") + str = exp.ReplaceAllString(str, "") + return str +} + +func parseAuthr(creator []string) []string { + exp1, _ := regexp.Compile("^(.*\\( *([^\\)]*) *\\))*$") + exp2, _ := regexp.Compile("^[^:]*: *(.*)$") + res := make([]string, len(creator)) + for i, s := range creator { + auth := exp1.FindStringSubmatch(s) + if auth != nil { + res[i] = cleanStr(strings.Join(auth[2:], ", ")) + } else { + auth := exp2.FindStringSubmatch(s) + if auth != nil { + res[i] = cleanStr(auth[1]) + } else { + res[i] = cleanStr(s) + } + } + } + return res +} + +func parseDescription(description []string) string { + str := cleanStr(strings.Join(description, "\n")) + str = strings.Replace(str, "

    ", "\n", -1) + exp, _ := regexp.Compile("<[^>]*>") + str = exp.ReplaceAllString(str, "") + str = strings.Replace(str, "&", "&", -1) + str = strings.Replace(str, "<", "<", -1) + str = strings.Replace(str, ">", ">", -1) + str = strings.Replace(str, "\\n", "\n", -1) + return str +} + +func parseSubject(subject []string) []string { + var res []string + for _, s := range subject { + res = append(res, strings.Split(s, " / ")...) + } + return res +} + +func parseDate(date []string) string { + if len(date) == 0 { + return "" + } + return strings.Replace(date[0], "Unspecified: ", "", -1) +} + +func keywords(b map[string]interface{}) (k []string) { + title, _ := b["title"].(string) + k = strings.Split(title, " ") + author, _ := b["author"].([]string) + for _, a := range author { + k = append(k, strings.Split(a, " ")...) + } + publisher, _ := b["publisher"].(string) + k = append(k, strings.Split(publisher, " ")...) + subject, _ := b["subject"].([]string) + k = append(k, subject...) + return +} From f1cfb8eea8aba3186cfb4f4ed58a5eb5110660f5 Mon Sep 17 00:00:00 2001 From: Las Zenow Date: Thu, 18 Jul 2013 22:12:43 +0200 Subject: [PATCH 13/13] Don't need to print the objectId --- tools/coverNew/coverNew.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/coverNew/coverNew.go b/tools/coverNew/coverNew.go index b283157..1931d05 100644 --- a/tools/coverNew/coverNew.go +++ b/tools/coverNew/coverNew.go @@ -13,7 +13,6 @@ func main() { for _, book := range books { fmt.Println(book.Title) - fmt.Println(book.File) e, err := OpenBook(book.File) if err != nil { fmt.Println("================", err)