diff --git a/.gitignore b/.gitignore index f4c6ca40..2adb13f9 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,4 @@ composer.lock www/manual/* !www/manual/index.php config/php/fpm/standardebooks.org-secrets.ini -www/bulk-downloads/months -www/bulk-downloads/subjects -www/bulk-downloads/collections +www/bulk-downloads/**/*.zip diff --git a/lib/Collection.php b/lib/Collection.php index 8d047dbb..479da80a 100644 --- a/lib/Collection.php +++ b/lib/Collection.php @@ -1,5 +1,7 @@ Name = $name; $this->Url = '/collections/' . Formatter::MakeUrlSafe($this->Name); } + + public function GetSortedName(): string{ + return preg_replace('/^(the|and|a|)\b/ius', '', $this->Name); + } } diff --git a/lib/Library.php b/lib/Library.php index b03f172a..b6fc46af 100644 --- a/lib/Library.php +++ b/lib/Library.php @@ -225,33 +225,50 @@ class Library{ return $ebooks; } - private static function FillBulkDownloadObject(string $file, string $downloadType): stdClass{ + private static function FillBulkDownloadObject(string $dir, string $downloadType): stdClass{ $obj = new stdClass(); - $obj->Size = Formatter::ToFileSize(filesize($file)); - $obj->Updated = new DateTime('@' . filemtime($file)); // The count of ebooks in each file is stored as a filesystem attribute - $obj->Count = exec('attr -g se-ebook-count ' . escapeshellarg($file)) ?: null; - if($obj->Count !== null){ - $obj->Count = intval($obj->Count); + $obj->EbookCount = exec('attr -g se-ebook-count ' . escapeshellarg($dir)) ?: null; + if($obj->EbookCount == null){ + $obj->EbookCount = 0; + } + else{ + $obj->EbookCount = intval($obj->EbookCount); } // The subject of the batch is stored as a filesystem attribute - $obj->Label = exec('attr -g se-label ' . escapeshellarg($file)) ?: null; + $obj->Label = exec('attr -g se-label ' . escapeshellarg($dir)) ?: null; if($obj->Label === null){ - $obj->Label = str_replace('se-ebooks-', '', basename($file, '.zip')); + $obj->Label = basename($dir); } $obj->UrlLabel = Formatter::MakeUrlSafe($obj->Label); - $obj->Url = '/bulk-downloads/' . $downloadType . '/' . $obj->UrlLabel . '/' . basename($file); - - // The type of ebook in the zip is stored as a filesystem attribute - $obj->Type = exec('attr -g se-ebook-type ' . escapeshellarg($file)); - if($obj->Type == 'epub-advanced'){ - $obj->Type = 'epub (advanced)'; + $obj->LabelSort = exec('attr -g se-label-sort ' . escapeshellarg($dir)) ?: null; + if($obj->LabelSort === null){ + $obj->LabelSort = basename($dir); } + $obj->ZipFiles = []; + + $files = glob($dir . '/*.zip'); + foreach($files as $file){ + $zipFile = new stdClass(); + $zipFile->Size = Formatter::ToFileSize(filesize($file)); + + $zipFile->Url = '/bulk-downloads/' . $downloadType . '/' . $obj->UrlLabel . '/' . basename($file); + + // The type of ebook in the zip is stored as a filesystem attribute + $zipFile->Type = exec('attr -g se-ebook-type ' . escapeshellarg($file)); + if($zipFile->Type == 'epub-advanced'){ + $zipFile->Type = 'epub (advanced)'; + } + + $obj->ZipFiles[] = $zipFile; + } + + $obj->Updated = new DateTime('@' . filemtime($files[0])); $obj->UpdatedString = $obj->Updated->format('M j'); // Add a period to the abbreviated month, but not if it's May (the only 3-letter month) $obj->UpdatedString = preg_replace('/^(.+?)(?UpdatedString); @@ -259,9 +276,16 @@ class Library{ $obj->UpdatedString = $obj->Updated->format('M j, Y'); } + // Sort the downloads by filename extension + $obj->ZipFiles = self::SortBulkDownloads($obj->ZipFiles); + return $obj; } + /** + * @param array $items + * @return array>> + */ private static function SortBulkDownloads(array $items): array{ // This sorts our items in a special order, epub first and advanced epub last $result = []; @@ -295,13 +319,17 @@ class Library{ public static function RebuildBulkDownloadsCache(): array{ $years = []; $subjects = []; + $collections = []; + $authors = []; // Generate bulk downloads by month - $files = glob(WEB_ROOT . '/bulk-downloads/months/*/*.zip'); - rsort($files); + // These get special treatment because they're sorted by two dimensions, + // year and month. + $dirs = glob(WEB_ROOT . '/bulk-downloads/months/*/', GLOB_NOSORT); + rsort($dirs); - foreach($files as $file){ - $obj = self::FillBulkDownloadObject($file, 'months'); + foreach($dirs as $dir){ + $obj = self::FillBulkDownloadObject($dir, 'months'); $date = new DateTime($obj->Label . '-01'); $year = $date->format('Y'); @@ -311,64 +339,36 @@ class Library{ $years[$year] = []; } - if(!isset($years[$year][$month])){ - $years[$year][$month] = []; - } - - $years[$year][$month][] = $obj; + $years[$year][$month] = $obj; } - // Sort the downloads by filename extension - foreach($years as $year => $months){ - foreach($months as $month => $items){ - $years[$year][$month] = self::SortBulkDownloads($items); - } - } - - apcu_store('bulk-downloads-years', $years); + apcu_store('bulk-downloads-years', $years, 43200); // 12 hours // Generate bulk downloads by subject - $files = glob(WEB_ROOT . '/bulk-downloads/subjects/*/*.zip'); - sort($files); - - foreach($files as $file){ - $obj = self::FillBulkDownloadObject($file, 'subjects'); - - if(!isset($subjects[$obj->UrlLabel])){ - $subjects[$obj->UrlLabel] = []; - } - - $subjects[$obj->UrlLabel][] = $obj; + foreach(glob(WEB_ROOT . '/bulk-downloads/subjects/*/', GLOB_NOSORT) as $dir){ + $subjects[] = self::FillBulkDownloadObject($dir, 'subjects'); } + usort($subjects, function($a, $b){ return $a->LabelSort <=> $b->LabelSort; }); - foreach($subjects as $subject => $items){ - $subjects[$subject] = self::SortBulkDownloads($items); - } - - apcu_store('bulk-downloads-subjects', $subjects); - + apcu_store('bulk-downloads-subjects', $subjects, 43200); // 12 hours // Generate bulk downloads by collection - $files = glob(WEB_ROOT . '/bulk-downloads/collections/*/*.zip'); - sort($files); - - foreach($files as $file){ - $obj = self::FillBulkDownloadObject($file, 'collections'); - - if(!isset($collections[$obj->UrlLabel])){ - $collections[$obj->UrlLabel] = []; - } - - $collections[$obj->UrlLabel][] = $obj; + foreach(glob(WEB_ROOT . '/bulk-downloads/collections/*/', GLOB_NOSORT) as $dir){ + $collections[] = self::FillBulkDownloadObject($dir, 'collections'); } + usort($collections, function($a, $b){ return $a->LabelSort <=> $b->LabelSort; }); - foreach($collections as $collection => $items){ - $collections[$collection] = self::SortBulkDownloads($items); + apcu_store('bulk-downloads-collections', $collections, 43200); // 12 hours + + // Generate bulk downloads by authors + foreach(glob(WEB_ROOT . '/bulk-downloads/authors/*/', GLOB_NOSORT) as $dir){ + $authors[] = self::FillBulkDownloadObject($dir, 'authors'); } + usort($authors, function($a, $b){ return $a->LabelSort <=> $b->LabelSort; }); - apcu_store('bulk-downloads-collections', $collections); + apcu_store('bulk-downloads-authors', $authors, 43200); // 12 hours - return ['years' => $years, 'subjects' => $subjects, 'collections' => $collections]; + return ['years' => $years, 'subjects' => $subjects, 'collections' => $collections, 'authors' => $authors]; } public static function RebuildCache(): void{ diff --git a/scripts/generate-bulk-downloads b/scripts/generate-bulk-downloads index b24bcdaf..ef01c46d 100755 --- a/scripts/generate-bulk-downloads +++ b/scripts/generate-bulk-downloads @@ -8,15 +8,12 @@ $longopts = ['webroot:']; $options = getopt('', $longopts); $webRoot = $options['webroot'] ?? WEB_ROOT; -$ebooksByMonth = []; -$lastUpdatedTimestampsByMonth = []; -$subjects = []; -$collections = []; -$ebooksBySubject = []; -$lastUpdatedTimestampsBySubject = []; -$lastUpdatedTimestampsByCollection = []; +$types = ['epub', 'epub-advanced', 'azw3', 'kepub', 'xhtml']; +$groups = ['collections', 'subjects', 'authors', 'months']; +$ebooksByGroup = []; +$updatedByGroup = []; -function CreateZip(string $filePath, array $ebooks, string $type, string $webRoot, string $label): void{ +function CreateZip(string $filePath, array $ebooks, string $type, string $webRoot): void{ $tempFilename = tempnam(sys_get_temp_dir(), "se-ebooks"); $zip = new ZipArchive(); @@ -59,20 +56,9 @@ function CreateZip(string $filePath, array $ebooks, string $type, string $webRoo $zip->close(); - $dir = dirname($filePath); - if(!is_dir($dir)){ - mkdir($dir, 0775, true); - } - rename($tempFilename, $filePath); - // Set a filesystem attribute for the number of ebooks in the file. This will be used - // to display that number on the downloads page. - exec('attr -q -s se-ebook-count -V ' . escapeshellarg(sizeof($ebooks)) . ' ' . escapeshellarg($filePath)); - exec('attr -q -s se-ebook-type -V ' . escapeshellarg($type) . ' ' . escapeshellarg($filePath)); - - exec('attr -q -s se-label -V ' . escapeshellarg($label) . ' ' . escapeshellarg($filePath)); } // Iterate over all ebooks and arrange them by publication month @@ -80,93 +66,103 @@ foreach(Library::GetEbooksFromFilesystem($webRoot) as $ebook){ $timestamp = $ebook->Created->format('Y-m'); $updatedTimestamp = $ebook->Updated->getTimestamp(); - if(!isset($ebooksByMonth[$timestamp])){ - $ebooksByMonth[$timestamp] = []; - $lastUpdatedTimestampsByMonth[$timestamp] = $updatedTimestamp; - } - // Add to the 'ebooks by month' list - $ebooksByMonth[$timestamp][] = $ebook; + if(!isset($ebooksByGroup['months'][$timestamp])){ + $obj = new stdClass(); + $obj->Label = $timestamp; + $obj->LabelSort = $timestamp; + $obj->Updated = $updatedTimestamp; + $obj->Ebooks = [$ebook]; - if($updatedTimestamp > $lastUpdatedTimestampsByMonth[$timestamp]){ - $lastUpdatedTimestampsByMonth[$timestamp] = $updatedTimestamp; + $ebooksByGroup['months'][$timestamp] = $obj; + } + else{ + $ebooksByGroup['months'][$timestamp]->Ebooks[] = $ebook; + if($updatedTimestamp > $ebooksByGroup['months'][$timestamp]->Updated){ + $ebooksByGroup['months'][$timestamp]->Updated = $updatedTimestamp; + } } // Add to the 'books by subject' list foreach($ebook->Tags as $tag){ - // Add the book's subjects to the main subjects list - if(!in_array($tag->Name, $subjects)){ - $subjects[] = $tag->Name; - $lastUpdatedTimestampsBySubject[$tag->Name] = $updatedTimestamp; + if(!isset($ebooksByGroup['subjects'][$tag->Name])){ + $obj = new stdClass(); + $obj->Label = $tag->Name; + $obj->LabelSort = $tag->Name; + $obj->Updated = $updatedTimestamp; + $obj->Ebooks = [$ebook]; + + $ebooksByGroup['subjects'][$tag->Name] = $obj; } - - // Sort this ebook by subject - $ebooksBySubject[$tag->Name][] = $ebook; - - if($updatedTimestamp > $lastUpdatedTimestampsBySubject[$tag->Name]){ - $lastUpdatedTimestampsBySubject[$tag->Name] = $updatedTimestamp; + else{ + $ebooksByGroup['subjects'][$tag->Name]->Ebooks[] = $ebook; + if($updatedTimestamp > $ebooksByGroup['subjects'][$tag->Name]->Updated){ + $ebooksByGroup['subjects'][$tag->Name]->Updated = $updatedTimestamp; + } } } // Add to the 'books by collection' list foreach($ebook->Collections as $collection){ - // Add the book's subjects to the main subjects list - if(!in_array($collection->Name, $collections)){ - $collections[] = $collection->Name; - $lastUpdatedTimestampsByCollection[$collection->Name] = $updatedTimestamp; + if(!isset($ebooksByGroup['collections'][$collection->Name])){ + $obj = new stdClass(); + $obj->Label = $collection->Name; + $obj->LabelSort = $collection->GetSortedName(); + $obj->Updated = $updatedTimestamp; + $obj->Ebooks = [$ebook]; + + $ebooksByGroup['collections'][$collection->Name] = $obj; } + else{ + $ebooksByGroup['collections'][$collection->Name]->Ebooks[] = $ebook; + if($updatedTimestamp > $ebooksByGroup['collections'][$collection->Name]->Updated){ + $ebooksByGroup['collections'][$collection->Name]->Updated = $updatedTimestamp; + } + } + } - // Sort this ebook by subject - $ebooksByCollection[$collection->Name][] = $ebook; + // Add to the 'books by author' list + foreach($ebook->Authors as $author){ + if(!isset($ebooksByGroup['authors'][$author->Name])){ + $obj = new stdClass(); + $obj->Label = $author->Name; + $obj->LabelSort = $author->SortName; + $obj->Updated = $updatedTimestamp; + $obj->Ebooks = [$ebook]; - if($updatedTimestamp > $lastUpdatedTimestampsByCollection[$collection->Name]){ - $lastUpdatedTimestampsByCollection[$collection->Name] = $updatedTimestamp; + $ebooksByGroup['authors'][$author->Name] = $obj; + } + else{ + $ebooksByGroup['authors'][$author->Name]->Ebooks[] = $ebook; + if($updatedTimestamp > $ebooksByGroup['authors'][$author->Name]->Updated){ + $ebooksByGroup['authors'][$author->Name]->Updated = $updatedTimestamp; + } } } } -$types = ['epub', 'epub-advanced', 'azw3', 'kepub', 'xhtml']; +foreach($groups as $group){ + foreach($ebooksByGroup[$group] as $collection){ + $urlSafeCollection = Formatter::MakeUrlSafe($collection->Label); + $parentDir = $webRoot . '/bulk-downloads/' . $group . '/' . $urlSafeCollection; -foreach($ebooksByMonth as $month => $ebooks){ - foreach($types as $type){ - $filename = 'se-ebooks-' . $month . '-' . $type . '.zip'; - $filePath = $webRoot . '/bulk-downloads/months/' . $month . '/' . $filename; + if(!is_dir($parentDir)){ + mkdir($parentDir, 0775, true); + } - // If the file doesn't exist, or if the content.opf last updated time is newer than the file modification time - if(!file_exists($filePath) || filemtime($filePath) < $lastUpdatedTimestampsByMonth[$month]){ - print('Creating ' . $filePath . "\n"); + exec('attr -q -s se-ebook-count -V ' . escapeshellarg(sizeof($collection->Ebooks)) . ' ' . escapeshellarg($parentDir)); + exec('attr -q -s se-label -V ' . escapeshellarg($collection->Label) . ' ' . escapeshellarg($parentDir)); + exec('attr -q -s se-label-sort -V ' . escapeshellarg($collection->LabelSort) . ' ' . escapeshellarg($parentDir)); - CreateZip($filePath, $ebooks, $type, $webRoot, $month); - } - } -} - -foreach($ebooksBySubject as $subject => $ebooks){ - foreach($types as $type){ - $urlSafeSubject = Formatter::MakeUrlSafe($subject); - $filename = 'se-ebooks-' . $urlSafeSubject . '-' . $type . '.zip'; - $filePath = $webRoot . '/bulk-downloads/subjects/' . $urlSafeSubject . '/'. $filename; - - // If the file doesn't exist, or if the content.opf last updated time is newer than the file modification time - if(!file_exists($filePath) || filemtime($filePath) < $lastUpdatedTimestampsBySubject[$subject]){ - print('Creating ' . $filePath . "\n"); - - CreateZip($filePath, $ebooks, $type, $webRoot, $subject); - } - } -} - -foreach($ebooksByCollection as $collection => $ebooks){ - foreach($types as $type){ - $urlSafeCollection = Formatter::MakeUrlSafe($collection); - $filename = 'se-ebooks-' . $urlSafeCollection . '-' . $type . '.zip'; - $filePath = $webRoot . '/bulk-downloads/collections/' . $urlSafeCollection . '/'. $filename; - - // If the file doesn't exist, or if the content.opf last updated time is newer than the file modification time - if(!file_exists($filePath) || filemtime($filePath) < $lastUpdatedTimestampsByCollection[$collection]){ - print('Creating ' . $filePath . "\n"); - - CreateZip($filePath, $ebooks, $type, $webRoot, $collection); + foreach($types as $type){ + $filePath = $parentDir . '/se-ebooks-' . $urlSafeCollection . '-' . $type . '.zip'; + + // If the file doesn't exist, or if the content.opf last updated time is newer than the file modification time + if(!file_exists($filePath) || filemtime($filePath) < $collection->Updated){ + print('Creating ' . $filePath . "\n"); + + CreateZip($filePath, $collection->Ebooks, $type, $webRoot); + } } } } diff --git a/templates/BulkDownloadTable.php b/templates/BulkDownloadTable.php index afde3254..ddaaa3da 100644 --- a/templates/BulkDownloadTable.php +++ b/templates/BulkDownloadTable.php @@ -8,13 +8,13 @@ - $items){ ?> + - Label) ?> - Count)) ?> - UpdatedString) ?> + Label) ?> + EbookCount)) ?> + UpdatedString) ?> - + ZipFiles as $item){ ?> Type ?> (Size) ?>) diff --git a/www/bulk-downloads/authors/index.php b/www/bulk-downloads/authors/index.php new file mode 100644 index 00000000..cec1ad2a --- /dev/null +++ b/www/bulk-downloads/authors/index.php @@ -0,0 +1,37 @@ + 'Downloads by Author', 'highlight' => '', 'description' => 'Download zip files containing all of the Standard Ebooks by a given author.']) ?> +
+
+

Downloads by Author

+ + $forbiddenException]) ?> + +

Patrons circle members can download zip files containing all of the ebooks that were released in a given month of Standard Ebooks history. You can join the Patrons Circle with a small donation in support of our continuing mission to create free, beautiful digital literature.

+

These zip files contain each ebook in every format we offer, and are updated once daily with the latest versions of each ebook.

+

If you’re a Patrons Circle member, when prompted enter your email address and leave the password field blank to download these files.

+ 'Author', 'collections' => $authors]); ?> +
+
+ diff --git a/www/bulk-downloads/collections/index.php b/www/bulk-downloads/collections/index.php new file mode 100644 index 00000000..f282cdef --- /dev/null +++ b/www/bulk-downloads/collections/index.php @@ -0,0 +1,37 @@ + 'Downloads by Collection', 'highlight' => '', 'description' => 'Download zip files containing all of the Standard Ebooks in a given collection.']) ?> +
+
+

Downloads by Collection

+ + $forbiddenException]) ?> + +

Patrons circle members can download zip files containing all of the ebooks that were released in a given month of Standard Ebooks history. You can join the Patrons Circle with a small donation in support of our continuing mission to create free, beautiful digital literature.

+

These zip files contain each ebook in every format we offer, and are updated once daily with the latest versions of each ebook.

+

If you’re a Patrons Circle member, when prompted enter your email address and leave the password field blank to download these files.

+ 'Collection', 'collections' => $collections]); ?> +
+
+ diff --git a/www/bulk-downloads/index.php b/www/bulk-downloads/index.php index c38988cd..dea1f6ec 100644 --- a/www/bulk-downloads/index.php +++ b/www/bulk-downloads/index.php @@ -1,7 +1,6 @@ 'Bulk Ebook Downloads', 'highlight' => '', 'description' => 'Download zip files containing all of the Standard Ebooks released in a given month.']) ?>
-
+

Bulk Ebook Downloads

@@ -41,54 +43,20 @@ catch(Safe\Exceptions\ApcuException $ex){ $forbiddenException]) ?>

Patrons circle members can download zip files containing all of the ebooks that were released in a given month of Standard Ebooks history. You can join the Patrons Circle with a small donation in support of our continuing mission to create free, beautiful digital literature.

-

These zip files contain each ebook in every format we offer, and are updated once daily with the latest versions of each ebook.

-

If you’re a Patrons Circle member, when prompted enter your email address and leave the password field blank to download these files.

- -
-

Downloads by subject

- 'Subject', 'collections' => $subjects]); ?> -
- -
-

Downloads by collection

- 'Collection', 'collections' => $collections]); ?> -
- -
-

Downloads by month

- - - $months){ - $yearHeader = Formatter::ToPlainText($year); - ?> - - - - - - - - - - - $items){ - $monthHeader = Formatter::ToPlainText($month); - ?> - - - - - - - - - - - - - -
MonthEbooksUpdatedEbook format
Count)) ?>UpdatedString) ?>Type ?>(Size) ?>)
-
+
diff --git a/www/bulk-downloads/months/index.php b/www/bulk-downloads/months/index.php new file mode 100644 index 00000000..f310acf8 --- /dev/null +++ b/www/bulk-downloads/months/index.php @@ -0,0 +1,69 @@ + 'Downloads by Month', 'highlight' => '', 'description' => 'Download zip files containing all of the Standard Ebooks released in a given month.']) ?> +
+
+

Downloads by Month

+ + $forbiddenException]) ?> + +

Patrons circle members can download zip files containing all of the ebooks that were released in a given month of Standard Ebooks history. You can join the Patrons Circle with a small donation in support of our continuing mission to create free, beautiful digital literature.

+

These zip files contain each ebook in every format we offer, and are updated once daily with the latest versions of each ebook.

+

If you’re a Patrons Circle member, when prompted enter your email address and leave the password field blank to download these files.

+ + + $months){ + $yearHeader = Formatter::ToPlainText($year); + ?> + + + + + + + + + + + $collection){ + $monthHeader = Formatter::ToPlainText($month); + ?> + + + + + + ZipFiles as $item){ ?> + + + + + + + + +
MonthEbooksUpdatedEbook format
EbookCount)) ?>UpdatedString) ?>Type ?>(Size) ?>)
+
+
+ diff --git a/www/bulk-downloads/subjects/index.php b/www/bulk-downloads/subjects/index.php new file mode 100644 index 00000000..5c927650 --- /dev/null +++ b/www/bulk-downloads/subjects/index.php @@ -0,0 +1,37 @@ + 'Downloads by Subject', 'highlight' => '', 'description' => 'Download zip files containing all of the Standard Ebooks by a given subject.']) ?> +
+
+

Downloads by Subject

+ + $forbiddenException]) ?> + +

Patrons circle members can download zip files containing all of the ebooks that were released in a given month of Standard Ebooks history. You can join the Patrons Circle with a small donation in support of our continuing mission to create free, beautiful digital literature.

+

These zip files contain each ebook in every format we offer, and are updated once daily with the latest versions of each ebook.

+

If you’re a Patrons Circle member, when prompted enter your email address and leave the password field blank to download these files.

+ 'Subject', 'collections' => $subjects]); ?> +
+
+ diff --git a/www/css/core.css b/www/css/core.css index 9272d1e4..f4d59bc0 100644 --- a/www/css/core.css +++ b/www/css/core.css @@ -737,6 +737,7 @@ ul.message.error li:only-child{ .download-list tbody .row-header{ font-weight: bold; + text-align: left; white-space: normal; } diff --git a/www/donate/index.php b/www/donate/index.php index 3cc58444..e89a7aa1 100644 --- a/www/donate/index.php +++ b/www/donate/index.php @@ -47,7 +47,7 @@ require_once('Core.php');
  • -

    Access to bulk ebook downloads to easily download entire months’ worth of ebooks at once.

    +

    Access to bulk ebook downloads to easily download whole collections of ebooks at once.

  • The ability to submit a book for inclusion on our Wanted Ebooks list, once per quarter. (Submissions must conform to our collections policy and are subject to approval.)