Change the way we deploy ebooks from destroying the entire apcu cache and rebuilding on the next web request, to rebuilding in the background

lib/Library.php

@@ -1,6 +1,8 @@
 <?
 use function Safe\apcu_fetch;
 use function Safe\preg_replace;
+use function Safe\touch;
+use function Safe\unlink;
 use function Safe\usort;
 
 class Library{
@@ -14,39 +16,19 @@ class Library{
 					$ebooks = apcu_fetch('ebooks-alpha');
 				}
 				catch(Safe\Exceptions\ApcuException $ex){
-					$ebooks = Library::GetEbooks();
+					Library::RebuildCache();
-
+					$ebooks = apcu_fetch('ebooks-alpha');
-					usort($ebooks, function($a, $b){
-						return strcmp(mb_strtolower($a->Authors[0]->SortName), mb_strtolower($b->Authors[0]->SortName));
-					});
-
-					apcu_store('ebooks-alpha', $ebooks);
 				}
 				break;
 
 			case SORT_NEWEST:
-				// Get all ebooks, sorted by newest first.
+				// Get all ebooks, sorted by release date first.
 				try{
 					$ebooks = apcu_fetch('ebooks-newest');
 				}
 				catch(Safe\Exceptions\ApcuException $ex){
-					$ebooks = Library::GetEbooks();
+					Library::RebuildCache();
-
+					$ebooks = apcu_fetch('ebooks-newest');
-					usort($ebooks, function($a, $b){
-						if($a->Timestamp < $b->Timestamp){
-							return -1;
-						}
-						elseif($a->Timestamp == $b->Timestamp){
-							return 0;
-						}
-						else{
-							return 1;
-						}
-					});
-
-					$ebooks = array_reverse($ebooks);
-
-					apcu_store('ebooks-newest', $ebooks);
 				}
 				break;
 
@@ -56,23 +38,8 @@ class Library{
 					$ebooks = apcu_fetch('ebooks-reading-ease');
 				}
 				catch(Safe\Exceptions\ApcuException $ex){
-					$ebooks = Library::GetEbooks();
+					Library::RebuildCache();
-
+					$ebooks = apcu_fetch('ebooks-reading-ease');
-					usort($ebooks, function($a, $b){
-						if($a->ReadingEase < $b->ReadingEase){
-							return -1;
-						}
-						elseif($a->ReadingEase == $b->ReadingEase){
-							return 0;
-						}
-						else{
-							return 1;
-						}
-					});
-
-					$ebooks = array_reverse($ebooks);
-
-					apcu_store('ebooks-reading-ease', $ebooks);
 				}
 				break;
 
@@ -82,21 +49,8 @@ class Library{
 					$ebooks = apcu_fetch('ebooks-length');
 				}
 				catch(Safe\Exceptions\ApcuException $ex){
-					$ebooks = Library::GetEbooks();
+					Library::RebuildCache();
-
+					$ebooks = apcu_fetch('ebooks-length');
-					usort($ebooks, function($a, $b){
-						if($a->WordCount < $b->WordCount){
-							return -1;
-						}
-						elseif($a->WordCount == $b->WordCount){
-							return 0;
-						}
-						else{
-							return 1;
-						}
-					});
-
-					apcu_store('ebooks-length', $ebooks);
 				}
 				break;
 
@@ -106,30 +60,8 @@ class Library{
 					$ebooks = apcu_fetch('ebooks');
 				}
 				catch(Safe\Exceptions\ApcuException $ex){
-					foreach(explode("\n", trim(shell_exec('find ' . EBOOKS_DIST_PATH . ' -name "content.opf"') ?? '')) as $filename){
+					Library::RebuildCache();
-						if(trim($filename) != ''){
+					$ebooks = apcu_fetch('ebooks');
-							$ebookWwwFilesystemPath = preg_replace('|/src/.+|ius', '', $filename) ?: '';
-							$ebook = null;
-							try{
-								$ebook = apcu_fetch('ebook-' . $ebookWwwFilesystemPath);
-							}
-							catch(Safe\Exceptions\ApcuException $ex){
-								try{
-									$ebook = new Ebook($ebookWwwFilesystemPath);
-									apcu_store('ebook-' . $ebookWwwFilesystemPath, $ebook);
-								}
-								catch(InvalidEbookException $ieEx){
-									// Do nothing if one specific ebook is causing problems
-								}
-							}
-
-							if($ebook !== null){
-								$ebooks[] = $ebook;
-							}
-						}
-					}
-
-					apcu_store('ebooks', $ebooks);
 				}
 				break;
 		}
@@ -139,32 +71,12 @@ class Library{
 
 	public static function GetEbooksByAuthor(string $wwwFilesystemPath): array{
 		// Do we have the author's ebooks cached?
+		$ebooks = [];
+
 		try{
 			$ebooks = apcu_fetch('author-' . $wwwFilesystemPath);
 		}
 		catch(Safe\Exceptions\ApcuException $ex){
-			$ebooks = [];
-
-			foreach(explode("\n", trim(shell_exec('find ' . escapeshellarg($wwwFilesystemPath) . ' -name "content.opf"') ?? '')) as $filename){
-				try{
-					$ebookWwwFilesystemPath = preg_replace('|/src/.+|ius', '', $filename) ?? '';
-					try{
-						$ebook = apcu_fetch('ebook-' . $ebookWwwFilesystemPath);
-					}
-					catch(Safe\Exceptions\ApcuException $ex){
-						$ebook = new Ebook($ebookWwwFilesystemPath);
-						apcu_store('ebook-' . $ebookWwwFilesystemPath, $ebook);
-					}
-
-					$ebooks[] = $ebook;
-
-				}
-				catch(\Exception $ex){
-					// An error in a book isn't fatal; just carry on.
-				}
-			}
-
-			apcu_store('author-' . $wwwFilesystemPath, $ebooks);
 		}
 
 		return $ebooks;
@@ -172,33 +84,12 @@ class Library{
 
 	public static function GetEbooksByTag(string $tag): array{
 		// Do we have the tag's ebooks cached?
+		$ebooks = [];
+
 		try{
 			$ebooks = apcu_fetch('tag-' . $tag);
 		}
 		catch(Safe\Exceptions\ApcuException $ex){
-			$ebooks = [];
-
-			foreach(explode("\n", trim(shell_exec('find ' . EBOOKS_DIST_PATH . ' -name "content.opf"') ?? '')) as $filename){
-				try{
-					$ebookWwwFilesystemPath = preg_replace('|/src/.+|ius', '', $filename) ?? '';
-					try{
-						$ebook = apcu_fetch('ebook-' . $ebookWwwFilesystemPath);
-					}
-					catch(Safe\Exceptions\ApcuException $ex){
-						$ebook = new Ebook($ebookWwwFilesystemPath);
-						apcu_store('ebook-' . $ebookWwwFilesystemPath, $ebook);
-					}
-
-					if($ebook->HasTag($tag)){
-						$ebooks[] = $ebook;
-					}
-				}
-				catch(\Exception $ex){
-					// An error in a book isn't fatal; just carry on.
-				}
-			}
-
-			apcu_store('tag-' . $tag, $ebooks);
 		}
 
 		return $ebooks;
@@ -206,33 +97,12 @@ class Library{
 
 	public static function GetEbooksByCollection(string $collection): array{
 		// Do we have the tag's ebooks cached?
+		$ebooks = [];
+
 		try{
 			$ebooks = apcu_fetch('collection-' . $collection);
 		}
 		catch(Safe\Exceptions\ApcuException $ex){
-			$ebooks = [];
-
-			foreach(explode("\n", trim(shell_exec('find ' . EBOOKS_DIST_PATH . ' -name "content.opf"') ?? '')) as $filename){
-				try{
-					$ebookWwwFilesystemPath = preg_replace('|/src/.+|ius', '', $filename) ?? '';
-					try{
-						$ebook = apcu_fetch('ebook-' . $ebookWwwFilesystemPath);
-					}
-					catch(Safe\Exceptions\ApcuException $ex){
-						$ebook = new Ebook($ebookWwwFilesystemPath);
-						apcu_store('ebook-' . $ebookWwwFilesystemPath, $ebook);
-					}
-
-					if($ebook->IsInCollection($collection)){
-						$ebooks[] = $ebook;
-					}
-				}
-				catch(\Exception $ex){
-					// An error in a book isn't fatal; just carry on.
-				}
-			}
-
-			apcu_store('collection-' . $collection, $ebooks);
 		}
 
 		return $ebooks;
@@ -250,4 +120,141 @@ class Library{
 
 		return $matches;
 	}
+
+	public static function RebuildCache(): void{
+		// We create and check a lockfile because this can be a long-running command.
+		// We don't want to queue up a bunch of these in case someone is refreshing the index constantly.
+		$lockfile = '/tmp/rebuild-library-cache-lock';
+		if(file_exists($lockfile)){
+			return;
+		}
+
+		touch($lockfile);
+
+		$ebooks = [];
+		$collections = [];
+		$tags = [];
+		$authors = [];
+
+		foreach(explode("\n", trim(shell_exec('find ' . EBOOKS_DIST_PATH . ' -name "content.opf"') ?? '')) as $filename){
+			try{
+				$ebookWwwFilesystemPath = preg_replace('|/src/.+|ius', '', $filename) ?? '';
+
+				$ebook = new Ebook($ebookWwwFilesystemPath);
+
+				$ebooks[$ebookWwwFilesystemPath] = $ebook;
+
+				// Create the collections cache
+				foreach($ebook->Collections as $collection){
+					$lcCollection = strtolower(Formatter::RemoveDiacritics($collection->Name));
+					if(!array_key_exists($lcCollection, $collections)){
+						$collections[$lcCollection] = [];
+					}
+
+					$collections[$lcCollection][] = $ebook;
+				}
+
+				// Create the tags cache
+				foreach($ebook->Tags as $tag){
+					$lcTag = strtolower($tag->Name);
+					if(!array_key_exists($lcTag, $tags)){
+						$tags[$lcTag] = [];
+					}
+
+					$tags[$lcTag][] = $ebook;
+				}
+
+				// Create the authors cache
+				$authorPath = EBOOKS_DIST_PATH . rtrim(preg_replace('|^/ebooks/|ius', '', $ebook->AuthorsUrl), '/');
+				if(!array_key_exists($authorPath, $authors)){
+					$authors[$authorPath] = [];
+				}
+
+				$authors[$authorPath][] = $ebook;
+			}
+			catch(\Exception $ex){
+				// An error in a book isn't fatal; just carry on.
+			}
+		}
+
+		apcu_clear_cache();
+
+		apcu_store('ebooks', $ebooks);
+
+		// Before we sort the list of ebooks and lose the array keys, store them by individual ebook
+		foreach($ebooks as $ebookWwwFilesystemPath => $ebook){
+			apcu_store('ebook-' . $ebookWwwFilesystemPath, $ebook);
+		}
+
+		// Sort ebooks by release date, then save
+		usort($ebooks, function($a, $b){
+			if($a->Timestamp < $b->Timestamp){
+				return -1;
+			}
+			elseif($a->Timestamp == $b->Timestamp){
+				return 0;
+			}
+			else{
+				return 1;
+			}
+		});
+
+		$ebooks = array_reverse($ebooks);
+
+		apcu_store('ebooks-newest', $ebooks);
+
+		// Sort ebooks by title alpha, then save
+		usort($ebooks, function($a, $b){
+			return strcmp(mb_strtolower($a->Authors[0]->SortName), mb_strtolower($b->Authors[0]->SortName));
+		});
+
+		apcu_store('ebooks-alpha', $ebooks);
+
+		// Sort ebooks by reading ease, then save
+		usort($ebooks, function($a, $b){
+			if($a->ReadingEase < $b->ReadingEase){
+				return -1;
+			}
+			elseif($a->ReadingEase == $b->ReadingEase){
+				return 0;
+			}
+			else{
+				return 1;
+			}
+		});
+
+		$ebooks = array_reverse($ebooks);
+
+		apcu_store('ebooks-reading-ease', $ebooks);
+
+		// Sort ebooks by word count, then save
+		usort($ebooks, function($a, $b){
+			if($a->WordCount < $b->WordCount){
+				return -1;
+			}
+			elseif($a->WordCount == $b->WordCount){
+				return 0;
+			}
+			else{
+				return 1;
+			}
+		});
+
+		apcu_store('ebooks-length', $ebooks);
+
+		// Now store various collections
+		foreach($collections as $collection => $ebooks){
+			apcu_store('collection-' . $collection, $ebooks);
+		}
+
+		foreach($tags as $tag => $ebooks){
+			apcu_store('tag-' . $tag, $ebooks);
+		}
+
+		foreach($authors as $author => $ebooks){
+			apcu_store('author-' . $author, $ebooks);
+		}
+
+		unlink($lockfile);
+	}
 }

scripts/deploy-ebook-to-www

@@ -200,10 +200,10 @@ do
 	sudo chmod --preserve-root --recursive g+ws "${webRoot}/www/images/covers/"
 
 	if [ "${verbose}" = "true" ]; then
-		printf "Flushing PHP-FPM opcache and apcu cache ... "
+		printf "Rebuilding web library cache ... "
 	fi
 
-	"${scriptsDir}"/reset-php-fpm-opcache standardebooks.org
+	"${scriptsDir}"/rebuild-library-cache
 
 	if [ "${verbose}" = "true" ]; then
 		printf "Done.\n"

scripts/rebuild-library-cache

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+usage(){
+	echo -n
+	fmt <<EOF
+DESCRIPTION
+	Rebuild the library cache stored in APCu for the standardebooks.org FPM pool.
+
+USAGE
+	rebuild-library-cache
+EOF
+	exit 1
+}
+
+if [ $# -eq 1 ]; then
+	if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
+		usage
+	fi
+fi
+
+# If this script is run by a user without sudo powers, they can be given for this command by creating a file in sudoers.d with:
+# MY_USERNAME ALL=(www-data) NOPASSWD: /usr/bin/env SCRIPT_FILENAME=/tmp/rebuild-library-cache.php REQUEST_METHOD=GET cgi-fcgi -bind -connect *
+
+echo "<?php require_once('Core.php'); Library::RebuildCache(); ?>" > /tmp/rebuild-library-cache.php
+sudo -u www-data env SCRIPT_FILENAME=/tmp/rebuild-library-cache.php REQUEST_METHOD=GET cgi-fcgi -bind -connect "/run/php/standardebooks.org.sock" &> /dev/null
+rm /tmp/rebuild-library-cache.php

scripts/reset-php-fpm-opcache

@@ -25,6 +25,6 @@ fi
 # If this script is run by a user without sudo powers, they can be given for this command by creating a file in sudoers.d with:
 # MY_USERNAME ALL=(www-data) NOPASSWD: /usr/bin/env SCRIPT_FILENAME=/tmp/php-fpm-opcache-reset.php REQUEST_METHOD=GET cgi-fcgi -bind -connect *
 
-echo '<?php opcache_reset(); if(function_exists("apcu_clear_cache")){ apcu_clear_cache(); } ?>' > /tmp/php-fpm-opcache-reset.php
+echo '<?php opcache_reset();?>' > /tmp/php-fpm-opcache-reset.php
 sudo -u www-data env SCRIPT_FILENAME=/tmp/php-fpm-opcache-reset.php REQUEST_METHOD=GET cgi-fcgi -bind -connect "/run/php/$1.sock" &> /dev/null
 rm /tmp/php-fpm-opcache-reset.php

www/ebooks/ebook.php

@@ -37,7 +37,6 @@ try{
 	}
 	catch(Safe\Exceptions\ApcuException $ex){
 		$ebook = new Ebook($wwwFilesystemPath);
-		apcu_store('ebook-' . $wwwFilesystemPath, $ebook);
 	}
 
 	// Generate the bottom carousel.

www/ebooks/index.php

@@ -42,10 +42,13 @@ try{
 	elseif($collection !== null){
 		$collection = strtolower(str_replace('-', ' ', Formatter::RemoveDiacritics($collection)));
 		$ebooks = Library::GetEbooksByCollection($collection);
-		// Get the *actual* name of the collection, in case there are accent marks (like "Arsène Lupin")
+
-		foreach($ebooks[0]->Collections as $c){
+		if(sizeof($ebooks) > 0){
-			if($collection == strtolower(str_replace('-', ' ', Formatter::RemoveDiacritics($c->Name)))){
+			// Get the *actual* name of the collection, in case there are accent marks (like "Arsène Lupin")
-				$collection = (string)$c->Name; // Explicit typecast to string to satisfy PHPStan
+			foreach($ebooks[0]->Collections as $c){
+				if($collection == strtolower(str_replace('-', ' ', Formatter::RemoveDiacritics($c->Name)))){
+					$collection = (string)$c->Name; // Explicit typecast to string to satisfy PHPStan
+				}
 			}
 		}
 		$collectionName = ucwords(preg_replace('/^The /ius', '', $collection) ?? '');