Fix broken updated timestamps in OPDS feeds, and fix and add some type hints.

This commit is contained in:
Alex Cabal 2024-11-08 12:43:47 -06:00
parent 34e35194d8
commit 06b82cdaaa
15 changed files with 344 additions and 300 deletions

View file

@ -126,14 +126,14 @@ if ! [ -d "${webRoot}" ]; then
die "${webRoot} does not exist or is not a directory."
fi
# Check for dependencies
# Check for dependencies.
require "convert" "Try: apt-get install imagemagick"
require "git" "Try: apt-get install git"
require "php" "Try: apt-get install php-cli"
require "rsvg-convert" "Try: apt-get install librsvg2-bin"
require "se" "Read: https://standardebooks.org/tools"
# cavif is compiled via Cargo: https://github.com/kornelski/cavif-rs
# `cavif` is compiled via Cargo, see <https://github.com/kornelski/cavif-rs>.
scriptsDir="$(dirname "$(readlink -f "$0")")"
@ -186,7 +186,7 @@ do
if [ "${lastPushHash}" != "" ]; then
# We were passed the hash of the last push before this one.
# Check to see if the cover image changed, to decide if we want to rebuild the cover image thumbnail/hero
# Check to see if the cover image changed, to decide if we want to rebuild the cover image thumbnail/hero.
diff=$(git diff "${lastPushHash}" HEAD)
if [ "${images}" = "true" ]; then
@ -199,8 +199,8 @@ do
fi
if [ "${build}" = "true" ]; then
# Check to see if the actual ebook changed, to decide if we want to build
# Always build if it doesn't exist
# Check to see if the actual ebook changed, to decide if we want to build.
# Always build if it doesn't exist.
if [[ ! -f "${webDir}/downloads/*.epub" ]] || [[ "${diff}" =~ diff\ --git\ a/src/ ]]; then
build="true"
else
@ -227,26 +227,26 @@ do
git show HEAD:images/cover.jpg > "${imgWorkDir}/${urlSafeIdentifier}.jpg"
git show HEAD:images/cover.svg > "${imgWorkDir}/${urlSafeIdentifier}.svg"
# We have to cd into the work dir, otherwise convert won't pick up the relative path of the jpg background in cover.svg
# We have to `cd` into the work dir, otherwise `convert` won't pick up the relative path of the jpg background in `cover.svg`.
pushd "${imgWorkDir}" > /dev/null || exit
# Build the hero image for individual ebook pages
# Resize and crop the image to 2156 width, 720 height, and starting at the coords 0,1078
# Build the hero image for individual ebook pages.
# Resize and crop the image to 2156 width, 720 height, and starting at the coords 0,1078.
convert -resize "1318" -crop "1318x439+0+659" -sampling-factor 4:2:0 -strip -quality 80 -colorspace RGB -interlace JPEG "${imgWorkDir}/${urlSafeIdentifier}.jpg" "${imgWorkDir}/${urlSafeIdentifier}-hero.jpg"
convert -resize "2636" -crop "2636x860+0+1318" -sampling-factor 4:2:0 -strip -quality 80 -colorspace RGB -interlace JPEG "${imgWorkDir}/${urlSafeIdentifier}.jpg" "${imgWorkDir}/${urlSafeIdentifier}-hero@2x.jpg"
"${scriptsDir}"/cavif --quiet --quality 50 "${imgWorkDir}/${urlSafeIdentifier}-hero.jpg" -o "${imgWorkDir}/${urlSafeIdentifier}-hero.avif"
"${scriptsDir}"/cavif --quiet --quality 50 "${imgWorkDir}/${urlSafeIdentifier}-hero@2x.jpg" -o "${imgWorkDir}/${urlSafeIdentifier}-hero@2x.avif"
# Build the cover image thumbnail
# Build the cover image thumbnail.
# We use JPG instead of SVG for several reasons:
# 1. A JPG is roughly 1/2 the file size of the same SVG, because the SVG must contain the JPG in base64 encoding
# 1. A JPG is roughly 1/2 the file size of the same SVG, because the SVG must contain the JPG in base64 encoding.
# 2. The "scale up" effect on mouse hover is blurry when used on SVGs.
sed -i "s/cover\.jpg/${urlSafeIdentifier}\.jpg/g" "${imgWorkDir}/${urlSafeIdentifier}.svg"
# Resize and compress the cover image (formula from Google Page Speed Insights)
# We can't use `convert` directly to do svg -> jpg, as sometimes it fails to load the linked cover.jpg within cover.svg.
# Resize and compress the cover image (formula from Google Page Speed Insights).
# We can't use `convert` directly to do svg -> jpg, as sometimes it fails to load the linked `cover.jpg` within `cover.svg`.
# So, we use `rsvg-convert` to write a png, then `convert` to convert and compress to jpg.
rsvg-convert --width 242 --keep-aspect-ratio --output "${imgWorkDir}/${urlSafeIdentifier}.png" "${imgWorkDir}/${urlSafeIdentifier}.svg"
rsvg-convert --width 484 --keep-aspect-ratio --output "${imgWorkDir}/${urlSafeIdentifier}@2x.png" "${imgWorkDir}/${urlSafeIdentifier}.svg"
@ -259,7 +259,7 @@ do
sudo chgrp --preserve-root --recursive "${group}" "${imgWorkDir}/${urlSafeIdentifier}"*
sudo chmod --preserve-root --recursive g+w "${imgWorkDir}/${urlSafeIdentifier}"*
# Remove unused images so we can copy the rest over with a glob
# Remove unused images so we can copy the rest over with a glob.
rm "${imgWorkDir}/${urlSafeIdentifier}".{png,jpg,svg}
if [ "${verbose}" = "true" ]; then
@ -278,7 +278,7 @@ do
mkdir -p "${workDir}/downloads"
# Build the ebook
# Build the ebook.
if [ "${epubcheck}" = "true" ]; then
if ! se build --output-dir="${workDir}"/downloads/ --check --kindle --kobo "${workDir}"; then
rm --preserve-root --recursive --force "${workDir}"
@ -291,7 +291,7 @@ do
fi
fi
# Build distributable covers
# Build distributable covers.
convert -resize "1400" -sampling-factor 4:2:0 -strip -quality 80 -colorspace RGB -interlace JPEG "${workDir}/src/epub/images/cover.svg" ""${workDir}"/downloads/cover.jpg"
convert -resize "350" -sampling-factor 4:2:0 -strip -quality 80 -colorspace RGB -interlace JPEG "${workDir}/src/epub/images/cover.svg" ""${workDir}"/downloads/cover-thumbnail.jpg"
@ -299,11 +299,10 @@ do
printf "Done.\n"
fi
# Get the book URL
# Get the book URL.
bookUrl=$(grep --only-matching --extended-regexp "<dc:identifier id=\"uid\">.+?</dc:identifier>" "${workDir}"/src/epub/content.opf | sed --regexp-extended "s/.*?url:https:\/\/standardebooks.org(.*?)<.*/\1/g")
# Get the last commit date so that we can update the modified timestamp in
# deployed content.opf. generate-opds uses this timestamp in its output.
# Get the last commit date so that we can update the modified timestamp in deployed `content.opf`. `generate-feeds` uses this timestamp in its output.
modifiedDate=$(TZ=UTC git log --date=iso-strict-local -1 --pretty=tformat:"%cd" --abbrev-commit | sed "s/+00:00/Z/")
sed --in-place --regexp-extended "s/<meta property=\"dcterms:modified\">.+?<\/meta>/<meta property=\"dcterms:modified\">${modifiedDate}<\/meta>/" "${workDir}/src/epub/content.opf"
@ -313,16 +312,16 @@ do
fi
# Recompose the epub into a single file, but put it outside of the epub src for now so we don't stomp on it with the following sections.
# We do this first because the tweaks below shouldn't apply to the single-page file
# We do this first because the tweaks below shouldn't apply to the single-page file.
se recompose-epub --xhtml --output "${workDir}"/single-page.xhtml --extra-css-file="${webRoot}/css/web.css" "${workDir}"
# Wrap book contents in a <main> tag
# Wrap book contents in a `<main>` element.
sed --in-place --regexp-extended "s|<body([^>]*)>|<body><main\1>|; s|</body>|</main></body>|" "${workDir}"/single-page.xhtml
# Add a navbar with a link back to the homepage
# Add a navbar with a link back to the homepage.
sed --in-place --regexp-extended "s|<body([^>]*)>|<body\1><header><nav><ul><li><a href=\"/\">Standard Ebooks</a></li><li><a href=\"${bookUrl}\">Back to ebook</a></li></ul></nav></header>|" "${workDir}"/single-page.xhtml
# Adjust sponsored links in the colophon
# Adjust sponsored links in the colophon.
sed --in-place 's|<p><a href="http|<p><a rel="nofollow" href="http|g' "${workDir}"/single-page.xhtml
if [ "${verbose}" = "true" ]; then
@ -330,60 +329,60 @@ do
fi
fi
# Make some compatibility adjustments for the individual XHTML files
# Make some compatibility adjustments for the individual XHTML files.
# Remove instances of the .xhtml filename extension in the source text
# Remove instances of the .xhtml filename extension in the source text.
find "${workDir}"/src/epub \( -type d -name .git -prune \) -o -type f -name "*.xhtml" -print0 | xargs -0 sed --in-place 's/\.xhtml//g'
# Add our web stylesheet to XHTML files
# Add our web stylesheet to XHTML files.
find "${workDir}"/src/epub \( -type d -name .git -prune \) -o -type f -name "*.xhtml" -print0 | xargs -0 sed --in-place --regexp-extended 's|</title>|</title>\n\t\t<link href="/css/web.css" media="screen" rel="stylesheet" type="text/css"/>|'
# Remove -epub-* CSS properties from CSS files as they're invalid in a web context
# Remove `-epub-*` CSS properties from CSS files as they're invalid in a web context.
sed --in-place --regexp-extended "s|\s*\-epub\-[^;]+?;||g" "${workDir}"/src/epub/css/*.css
# Add lang attributes
# Add lang attributes.
find "${workDir}"/src/epub \( -type d -name .git -prune \) -o -type f -name "*.xhtml" -print0 | xargs -0 sed --in-place --regexp-extended 's/xml:lang="([^"]+?)"/xml:lang="\1" lang="\1"/g'
# Add the work title to <title> tags in the source text
# Add the work title to <title> tags in the source text.
workTitle=$(grep --only-matching --extended-regexp "<dc:title id=\"title\">(.+?)</dc:title>" "${workDir}"/src/epub/content.opf | sed --regexp-extended "s/<[^>]+?>//g")
find "${workDir}"/src/epub \( -type d -name .git -prune \) -o -type f -name "*.xhtml" -print0 | xargs -0 sed --in-place --regexp-extended "s|<title>|<title>${workTitle} - |g"
# Wrap book contents in a <main> tag
# Wrap book contents in a `<main>` element.
find "${workDir}"/src/epub \( -type d -name .git -prune \) -o -type f -name "*.xhtml" -print0 | xargs -0 sed --in-place --regexp-extended "s|<body([^>]*)>|<body><main\1>|; s|</body>|</main></body>|"
# Add the header nav to each page
# Add the header nav to each page.
find "${workDir}"/src/epub \( -type d -name .git -prune \) -o -type f -name "*.xhtml" -print0 | xargs -0 sed --in-place --regexp-extended "s|<body([^>]*)>|<body\1><header><nav><ul><li><a href=\"/\">Standard Ebooks</a></li><li><a href=\"${bookUrl}\">Back to ebook</a></li><li><a href=\"${bookUrl}/text\">Table of contents</a></li></ul></nav></header>|"
# Add a chapter navigation footer to each page
# Add a chapter navigation footer to each page.
"${scriptsDir}"/inject-chapter-navigation-footer "${workDir}" "${bookUrl}"
# Adjust sponsored links in the colophon
# Adjust sponsored links in the colophon.
sed --in-place 's|<p><a href="http|<p><a rel="nofollow" href="http|g' "${workDir}"/src/epub/text/colophon.xhtml
# Done adding compatibility!
if [ "${recompose}" = "true" ]; then
# Move the single-page file back into the /src/epub/text/ folder
# Move the single-page file back into the `/src/epub/text/` folder.
mv "${workDir}"/single-page.xhtml "${workDir}"/src/epub/text/single-page.xhtml
fi
# Add viewport meta elements to all output
# Add viewport meta elements to all output.
find "${workDir}" \( -type d -name .git -prune \) -o -type f -name "*.xhtml" -print0 | xargs -0 sed --in-place "s|</title>|</title>\n\t\t<meta content=\"width=device-width, initial-scale=1\" name=\"viewport\"/>|"
# Delete the contents of the old webdir
# Delete the contents of the old webdir.
rm --preserve-root --recursive --force "${webDir}"
# Re-create the webdir
# Re-create the webdir.
mkdir -p "${webDir}"
# Move contents of the work dir over
# Move contents of the work dir over.
mv "${workDir}"/downloads "${webDir}/"
rm "${workDir}/src/epub/onix.xml"
mv "${workDir}"/src/epub/* "${webDir}/"
fi
if [ "${images}" = "true" ]; then
# Move the cover images over
# Move the cover images over.
mv "${imgWorkDir}/${urlSafeIdentifier}"*.{jpg,avif} "${webRoot}/images/covers/"
fi
@ -397,7 +396,7 @@ do
printf "Done.\n"
fi
# Delete the now-empty work dir (empty except for .git)
# Delete the now-empty work dir (empty except for `.git`).
rm --preserve-root --recursive --force "${workDir}" "${imgWorkDir}"
sudo chgrp --preserve-root --recursive "${group}" "${webDir}"
@ -415,7 +414,7 @@ if tsp | grep --quiet --extended-regexp "^[0-9]+\s+queued"; then
fi
if [ "${feeds}" = "true" ]; then
# Build the various feeds catalog, but only if we don't have more items in the tsp build queue.
# Build the various feeds catalog, but only if we don't have more items in the `tsp` build queue.
if [ "${queuedTasks}" = "false" ]; then
if [ "${verbose}" = "true" ]; then
printf "Building feeds ... "
@ -435,7 +434,7 @@ if [ "${feeds}" = "true" ]; then
fi
if [ "${bulkDownloads}" = "true" ]; then
# Build the various feeds catalog, but only if we don't have more items in the tsp build queue.
# Build the various feeds catalog, but only if we don't have more items in the `tsp` build queue.
if [ "${queuedTasks}" = "false" ]; then
if [ "${verbose}" = "true" ]; then
printf "Building bulk downloads ... "

View file

@ -21,11 +21,10 @@ $webRoot = $options['webroot'] ?? WEB_ROOT;
$types = ['epub', 'epub-advanced', 'azw3', 'kepub', 'xhtml'];
$groups = ['collections', 'subjects', 'authors', 'months'];
$ebooksByGroup = [];
$updatedByGroup = [];
$ebooksByGroup = ['collections' => [], 'subjects' => [], 'authors' => [], 'months' => []];
function rrmdir(string $src): void{
// See https://www.php.net/manual/en/function.rmdir.php#117354
// See <https://www.php.net/manual/en/function.rmdir.php#117354>.
$dir = opendir($src);
while(false !== ($file = readdir($dir))){
if (($file != '.') && ($file != '..')){
@ -103,12 +102,12 @@ function CreateZip(string $filePath, array $ebooks, string $type, string $webRoo
exec('attr -q -s se-ebook-type -V ' . escapeshellarg($type) . ' ' . escapeshellarg($filePath));
}
// Iterate over all ebooks and arrange them by publication month
// Iterate over all ebooks and arrange them by publication month.
foreach(Library::GetEbooks() as $ebook){
$timestamp = $ebook->EbookCreated->format('Y-m');
$updatedTimestamp = $ebook->EbookUpdated->getTimestamp();
// Add to the 'ebooks by month' list
// Add to the 'ebooks by month' list.
if(!isset($ebooksByGroup['months'][$timestamp])){
$obj = new stdClass();
$obj->Label = $timestamp;
@ -126,7 +125,7 @@ foreach(Library::GetEbooks() as $ebook){
}
}
// Add to the 'books by subject' list
// Add to the 'books by subject' list.
foreach($ebook->Tags as $tag){
if(!isset($ebooksByGroup['subjects'][$tag->Name])){
$obj = new stdClass();
@ -146,7 +145,7 @@ foreach(Library::GetEbooks() as $ebook){
}
}
// Add to the 'books by collection' list
// Add to the 'books by collection' list.
foreach($ebook->CollectionMemberships as $cm){
$collection = $cm->Collection;
if(!isset($ebooksByGroup['collections'][$collection->Name])){
@ -167,7 +166,7 @@ foreach(Library::GetEbooks() as $ebook){
}
}
// Add to the 'books by author' list
// Add to the 'books by author' list.
// We have to index by UrlName for cases like `Samuel Butler` whose UrlName is `samuel-butler-1612-1680`.
$authorsUrl = preg_replace('|^/ebooks/|', '', $ebook->AuthorsUrl);
if(!isset($ebooksByGroup['authors'][$authorsUrl])){
@ -189,7 +188,7 @@ foreach(Library::GetEbooks() as $ebook){
}
foreach($groups as $group){
// First delete any orphan directories that we don't expect to be here, for example a collection that was later renamed
// First delete any orphan directories that we don't expect to be here, for example a collection that was later renamed.
foreach(glob($webRoot . '/bulk-downloads/' . $group . '/*/') as $dir){
$expected = false;
foreach($ebooksByGroup[$group] as $collection){
@ -217,13 +216,13 @@ foreach($groups as $group){
exec('attr -q -s se-label -V ' . escapeshellarg($collection->Label) . ' ' . escapeshellarg($parentDir));
exec('attr -q -s se-label-sort -V ' . escapeshellarg($collection->LabelSort) . ' ' . escapeshellarg($parentDir));
// We also need to save the URL label for author edge cases like `Samuel Butler` -> `samuel-butler-1612-1680` or `Karl Marx and Freidrich Engels` -> `karl-marx_friedrich-engels`
// We also need to save the URL label for author edge cases like `Samuel Butler` -> `samuel-butler-1612-1680` or `Karl Marx and Freidrich Engels` -> `karl-marx_friedrich-engels`.
exec('attr -q -s se-url-label -V ' . escapeshellarg($collection->UrlLabel) . ' ' . escapeshellarg($parentDir));
foreach($types as $type){
$filePath = $parentDir . '/se-ebooks-' . $collection->UrlLabel . '-' . $type . '.zip';
// If the file doesn't exist, or if the content.opf last updated time is newer than the file modification time
// If the file doesn't exist, or if the `content.opf` last updated time is newer than the file modification time.
if(!file_exists($filePath) || filemtime($filePath) < $collection->Updated){
print('Creating ' . $filePath . "\n");
@ -233,8 +232,8 @@ foreach($groups as $group){
}
}
// Set ownership and permissions
// We don't use PHP's built in chown/chmod chmod can't accept strings
// The `chmod +X` command, with a capital X, makes only matched directories executable.
// Set ownership and permissions.
// We don't use PHP's built in `chown()`/`chmod()` because `chmod()` can't accept strings.
// The `chmod +X` command, with a capital `X`, makes only matched directories executable.
exec('sudo chown --preserve-root --recursive se:committers ' . escapeshellarg($webRoot) . '/bulk-downloads/*/');
exec('sudo chmod --preserve-root --recursive a+r,ug+w,a+X ' . escapeshellarg($webRoot) . '/bulk-downloads/*/');

View file

@ -13,7 +13,7 @@ function SortByUpdatedDesc(Ebook $a, Ebook $b): int{
return $b->EbookUpdated <=> $a->EbookUpdated;
}
function SaveFeed(Feed $feed, bool $force, ?string $label = null, ?string $labelSort = null, DateTimeImmutable $now = null): void{
function SaveFeed(Feed $feed, bool $force, ?string $label = null, ?string $labelSort = null, ?DateTimeImmutable $now = null): void{
$updateAttrs = false;
if($force){
@ -35,11 +35,24 @@ function SaveFeed(Feed $feed, bool $force, ?string $label = null, ?string $label
/**
* @param array<string, array<string, string>> $collections
* @param array<Ebook> $ebooks
* @param array<string, array<Ebook>> $ebooks
*/
function CreateOpdsCollectionFeed(string $name, string $url, string $description, array $collections, array $ebooks, DateTimeImmutable $now, string $webRoot, OpdsNavigationFeed $opdsRoot, bool $force): void{
$collator = collator_create('en_US'); // Used for sorting letters with diacritics like in author names
usort($collections, function($a, $b) use($collator){ return $collator->compare($a['sortedname'], $b['sortedname']); });
$collator = Collator::create('en_US'); // Used for sorting letters with diacritics like in author names
if($collator === null){
return;
}
usort($collections, function($a, $b) use($collator){
$result = $collator->compare($a['sortedname'], $b['sortedname']);
if($result === false){
return 0;
}
else{
return $result;
}
});
// Create the collections navigation document.
$collectionNavigationEntries = [];
@ -108,7 +121,7 @@ foreach(Library::GetEbooks() as $ebook){
$authorsUrl = preg_replace('|^/ebooks/|', '', $ebook->AuthorsUrl);
$ebooksByAuthor[$authorsUrl][] = $ebook;
$authors[$authorsUrl] = ['id' => $authorsUrl, 'name' => strip_tags($ebook->AuthorsHtml), 'sortedname' => $ebook->Authors[0]->SortName];
$authors[$authorsUrl] = ['id' => $authorsUrl, 'name' => strip_tags($ebook->AuthorsHtml), 'sortedname' => $ebook->Authors[0]->SortName ?? $ebook->Authors[0]->Name];
}
usort($allEbooks, 'SortByUpdatedDesc');

View file

@ -16,8 +16,8 @@ use Facebook\WebDriver\WebDriverExpectedCondition;
use Facebook\WebDriver\Remote\DesiredCapabilities;
use Facebook\WebDriver\Firefox\FirefoxDriver;
use Facebook\WebDriver\Firefox\FirefoxOptions;
use Facebook\WebDriver\WebDriverElement;
use Safe\DateTimeImmutable;
use function Safe\file_get_contents;
use function Safe\file_put_contents;
use function Safe\preg_replace;
@ -119,8 +119,11 @@ try{
$log->Write('Logging in to Fractured Atlas ...');
// We were redirected to the login page, so try to log in.
/** @var WebDriverElement $emailField */
$emailField = $driver->wait(20, 250)->until(WebDriverExpectedCondition::visibilityOfElementLocated(WebDriverBy::xpath('//input[@type="email"]')));
/** @var WebDriverElement $passwordField */
$passwordField = $driver->wait(20, 250)->until(WebDriverExpectedCondition::visibilityOfElementLocated(WebDriverBy::xpath('//input[@type="password"]')));
/** @var WebDriverElement $submitButton */
$submitButton = $driver->wait(20, 250)->until(WebDriverExpectedCondition::visibilityOfElementLocated(WebDriverBy::xpath('//button[@type="submit"]')));
// Fill out and submit the form.
@ -150,7 +153,9 @@ try{
for($i = 0; $i < sizeof($elements); $i++){
$td = $elements[$i];
$transactionId = trim($td->getDomProperty('textContent'));
/** @var string $transactionId */
$transactionId = $td->getDomProperty('textContent');
$transactionId = trim($transactionId);
if($transactionId === ''){
continue;
@ -172,7 +177,9 @@ try{
for($i = 0; $i < sizeof($elements); $i++){
$td = $elements[$i];
$transactionId = trim($td->getDomProperty('textContent'));
/** @var string $transactionId */
$transactionId = $td->getDomProperty('textContent');
$transactionId = trim($transactionId);
if($transactionId === ''){
continue;

View file

@ -7,6 +7,7 @@ use Facebook\WebDriver\WebDriverExpectedCondition;
use Facebook\WebDriver\Remote\DesiredCapabilities;
use Facebook\WebDriver\Firefox\FirefoxDriver;
use Facebook\WebDriver\Firefox\FirefoxOptions;
use Facebook\WebDriver\WebDriverElement;
use Safe\DateTimeImmutable;
use function Safe\preg_match;
@ -69,171 +70,215 @@ try{
foreach($pendingPayments as $pendingPayment){
$pendingPayment->Processor = PaymentProcessorType::from($pendingPayment->Processor);
if($pendingPayment->Processor == PaymentProcessorType::FracturedAtlas){
$log->Write('Processing donation ' . $pendingPayment->TransactionId . ' ...');
switch($pendingPayment->Processor){
case PaymentProcessorType::FracturedAtlas:
$log->Write('Processing donation ' . $pendingPayment->TransactionId . ' ...');
if(Db::QueryBool('
SELECT exists(
select *
from Payments
where TransactionId = ?
)
', [$pendingPayment->TransactionId])){
$log->Write('Donation already exists in database.');
continue;
}
$driver->get('https://fundraising.fracturedatlas.org/admin/donations?query=' . $pendingPayment->TransactionId);
// Check if we need to log in to FA.
// Wait until the <body> element is visible, then check the current URL.
$driver->wait(20, 250)->until(WebDriverExpectedCondition::visibilityOfElementLocated(WebDriverBy::xpath('/html/body')));
if(stripos($driver->getCurrentUrl(), 'auth0.com')){
$log->Write('Logging in to Fractured Atlas ...');
// We were redirected to the login page, so try to log in.
$emailField = $driver->wait(20, 250)->until(WebDriverExpectedCondition::visibilityOfElementLocated(WebDriverBy::xpath('//input[@type="email"]')));
$passwordField = $driver->wait(20, 250)->until(WebDriverExpectedCondition::visibilityOfElementLocated(WebDriverBy::xpath('//input[@type="password"]')));
$submitButton = $driver->wait(20, 250)->until(WebDriverExpectedCondition::visibilityOfElementLocated(WebDriverBy::xpath('//button[@type="submit"]')));
// Fill out and submit the form.
$emailField->sendKeys($faUsername);
$passwordField->sendKeys($faPassword);
$submitButton->click();
}
// Wait until the page finishes loading.
// We have to expand the row before we can select its contents, so click the 'expand' button once it's visible.
try{
$toggleButton = $driver->wait(20, 250)->until(WebDriverExpectedCondition::visibilityOfElementLocated(WebDriverBy::xpath('//button[contains(@class, "button-toggle")]')));
}
catch(Exception){
$log->Write('Error: Couldn\'t find donation.');
continue;
}
$toggleButton->click();
// Our target row is now visible, extract the data!
// In the FA donations table, there is a header row, and an expandable details row. The header row tells us if the donation is recurring, and the details row has the rest of the information.
$detailsRow = $driver->wait(20, 250)->until(WebDriverExpectedCondition::visibilityOfElementLocated(WebDriverBy::xpath('//tr[starts-with(@id, "expanded") and contains(@id, "' . $pendingPayment->TransactionId . '")]')));
$headerRow = $driver->findElement(WebDriverBy::xpath('//tr[not(starts-with(@id, "expanded")) and contains(@id, "' . $pendingPayment->TransactionId . '")]'));
$payment = new Payment();
$payment->User = new User();
$payment->Processor = $pendingPayment->Processor;
$hasSoftCredit = false;
try{
// If the donation is via a foundation (like American Online Giving Foundation) then there will be a 'soft credit' <th> element.
if(sizeof($detailsRow->findElements(WebDriverBy::xpath('//th[normalize-space(.) = "Soft Credit Donor Info"]'))) > 0){
// We're a foundation donation
$payment->User->Name = trim($detailsRow->findElement(WebDriverBy::xpath('//td[preceding-sibling::th[normalize-space(.) = "Name"] and (ancestor::tbody[1])[(./preceding-sibling::thead[1])//th[normalize-space(.) = "Soft Credit Donor Info"]]]'))->getText());
$payment->User->Email = trim($detailsRow->findElement(WebDriverBy::xpath('//td[preceding-sibling::th[normalize-space(.) = "Email"] and (ancestor::tbody[1])[(./preceding-sibling::thead[1])//th[normalize-space(.) = "Soft Credit Donor Info"]]]'))->getText());
$hasSoftCredit = true;
}
else{
// We're a regular donation
$payment->User->Name = trim($detailsRow->findElement(WebDriverBy::xpath('//td[preceding-sibling::th[normalize-space(.) = "Name"]]'))->getText());
$payment->User->Email = trim($detailsRow->findElement(WebDriverBy::xpath('//td[preceding-sibling::th[normalize-space(.) = "Email"]]'))->getText());
}
// These donations are typically (always?) employer matches.
// FA does not provide a way to connect the original donation with the employer match.
// Example bbf87b83-d341-426f-b6c9-9091e3222e57
if($payment->User->Name == 'American Online Giving Foundation'){
$payment->IsMatchingDonation = true;
}
// We can get here via an AOGF donation that is anonymous.
if(!$hasSoftCredit && ($payment->User->Email == 'Not provided' || $payment->User->Email == '')){
$payment->User = null;
}
}
catch(Exception){
// Anonymous donations don't have these elements present and will throw an exception.
$payment->User = null;
}
$payment->Created = DateTimeImmutable::createFromFormat('n/j/Y', trim($detailsRow->findElement(WebDriverBy::xpath('//td[preceding-sibling::th[normalize-space(.) = "Created"]]'))->getText()));
$payment->IsRecurring = sizeof($headerRow->findElements(WebDriverBy::xpath('//td[contains(., "Recurring")]'))) > 0;
$payment->Amount = floatval(preg_replace('/[^0-9\.]/', '', trim($detailsRow->findElement(WebDriverBy::xpath('//td[preceding-sibling::th[normalize-space(.) = "Total Amount"]]'))->getText())));
$payment->Fee = floatval(preg_replace('/[^0-9\.]/', '', trim($detailsRow->findElement(WebDriverBy::xpath('//td[preceding-sibling::th[normalize-space(.) = "Fee"]]'))->getText())));
$transactionId = (string)($detailsRow->findElement(WebDriverBy::xpath('//td[preceding-sibling::th[normalize-space(.) = "ID"]]'))->getText());
$transactionId = str_replace('View on FS', '', $transactionId);
$transactionId = str_replace('View on Finance', '', $transactionId);
$payment->TransactionId = trim($transactionId);
// We might also get a case where the donation is on behalf of a company match, but there's not really a way to distinguish that. Do a rough check.
// See donation 00b60a22-eafa-44cb-9850-54bef9763e8d
if($payment->User !== null && !$hasSoftCredit && preg_match('/\b(L\.?L\.?C\.?|Foundation|President|Fund|Charitable)\b/ius', $payment->User->Name ?? '')){
$payment->User = null;
}
// All set - create the payment.
try{
$payment->Create();
}
catch(Exceptions\PaymentExistsException){
// Payment already exists, just continue.
$log->Write('Donation already in database.');
continue;
}
// Does this payment create a new Patron in the Patrons Circle?
// If the user is *already* a Patron, then we just create the payment without further processing.
if(
(
$payment->IsRecurring
&&
$payment->Amount >= 10
&&
$payment->Created >= $lastMonth
)
||
(
!$payment->IsRecurring
&&
$payment->Amount >= 100
&&
$payment->Created >= $lastYear
)
){
// This payment is eligible for the Patrons Circle!
if($payment->User !== null){
// Are we already a patron?
if(!Db::QueryBool('
if(Db::QueryBool('
SELECT exists(
select *
from Patrons
where UserId = ?
and Ended is null
from Payments
where TransactionId = ?
)
', [$payment->UserId])){
// Not a patron yet, add them to the Patrons Circle.
', [$pendingPayment->TransactionId])){
$log->Write('Donation already exists in database.');
continue;
}
$patron = new Patron();
$patron->UserId = $payment->UserId;
$patron->User = $payment->User;
$patron->User->Payments = [$payment];
$patron->IsAnonymous = (trim($detailsRow->findElement(WebDriverBy::xpath('//td[preceding-sibling::th[normalize-space(.) = "Attribution"]]'))->getText()) == 'Private');
$patron->IsSubscribedToEmails = $patron->User !== null && $patron->User->Email !== null;
$driver->get('https://fundraising.fracturedatlas.org/admin/donations?query=' . $pendingPayment->TransactionId);
try{
$patron->AlternateName = trim($detailsRow->findElement(WebDriverBy::xpath('//td[preceding-sibling::th[normalize-space(.) = "Attribution Text"]]'))->getText());
}
catch(Exception){
// Check if we need to log in to FA.
// Wait until the <body> element is visible, then check the current URL.
$driver->wait(20, 250)->until(WebDriverExpectedCondition::visibilityOfElementLocated(WebDriverBy::xpath('/html/body')));
if(stripos($driver->getCurrentUrl(), 'auth0.com')){
$log->Write('Logging in to Fractured Atlas ...');
// We were redirected to the login page, so try to log in.
/** @var WebDriverElement $emailField */
$emailField = $driver->wait(20, 250)->until(WebDriverExpectedCondition::visibilityOfElementLocated(WebDriverBy::xpath('//input[@type="email"]')));
/** @var WebDriverElement $passwordField */
$passwordField = $driver->wait(20, 250)->until(WebDriverExpectedCondition::visibilityOfElementLocated(WebDriverBy::xpath('//input[@type="password"]')));
/** @var WebDriverElement $submitButton */
$submitButton = $driver->wait(20, 250)->until(WebDriverExpectedCondition::visibilityOfElementLocated(WebDriverBy::xpath('//button[@type="submit"]')));
// Fill out and submit the form.
$emailField->sendKeys($faUsername);
$passwordField->sendKeys($faPassword);
$submitButton->click();
}
// Wait until the page finishes loading.
// We have to expand the row before we can select its contents, so click the 'expand' button once it's visible.
try{
/** @var WebDriverElement $toggleButton */
$toggleButton = $driver->wait(20, 250)->until(WebDriverExpectedCondition::visibilityOfElementLocated(WebDriverBy::xpath('//button[contains(@class, "button-toggle")]')));
}
catch(Exception){
$log->Write('Error: Couldn\'t find donation.');
continue;
}
$toggleButton->click();
// Our target row is now visible, extract the data!
// In the FA donations table, there is a header row, and an expandable details row. The header row tells us if the donation is recurring, and the details row has the rest of the information.
/** @var WebDriverElement $detailsRow */
$detailsRow = $driver->wait(20, 250)->until(WebDriverExpectedCondition::visibilityOfElementLocated(WebDriverBy::xpath('//tr[starts-with(@id, "expanded") and contains(@id, "' . $pendingPayment->TransactionId . '")]')));
$headerRow = $driver->findElement(WebDriverBy::xpath('//tr[not(starts-with(@id, "expanded")) and contains(@id, "' . $pendingPayment->TransactionId . '")]'));
$payment = new Payment();
$payment->User = new User();
$payment->Processor = $pendingPayment->Processor;
$hasSoftCredit = false;
try{
// If the donation is via a foundation (like American Online Giving Foundation) then there will be a 'soft credit' <th> element.
if(sizeof($detailsRow->findElements(WebDriverBy::xpath('//th[normalize-space(.) = "Soft Credit Donor Info"]'))) > 0){
// We're a foundation donation
$payment->User->Name = trim($detailsRow->findElement(WebDriverBy::xpath('//td[preceding-sibling::th[normalize-space(.) = "Name"] and (ancestor::tbody[1])[(./preceding-sibling::thead[1])//th[normalize-space(.) = "Soft Credit Donor Info"]]]'))->getText());
$payment->User->Email = trim($detailsRow->findElement(WebDriverBy::xpath('//td[preceding-sibling::th[normalize-space(.) = "Email"] and (ancestor::tbody[1])[(./preceding-sibling::thead[1])//th[normalize-space(.) = "Soft Credit Donor Info"]]]'))->getText());
$hasSoftCredit = true;
}
else{
// We're a regular donation
$payment->User->Name = trim($detailsRow->findElement(WebDriverBy::xpath('//td[preceding-sibling::th[normalize-space(.) = "Name"]]'))->getText());
$payment->User->Email = trim($detailsRow->findElement(WebDriverBy::xpath('//td[preceding-sibling::th[normalize-space(.) = "Email"]]'))->getText());
}
// These donations are typically (always?) employer matches.
// FA does not provide a way to connect the original donation with the employer match.
// Example bbf87b83-d341-426f-b6c9-9091e3222e57
if($payment->User->Name == 'American Online Giving Foundation'){
$payment->IsMatchingDonation = true;
}
// We can get here via an AOGF donation that is anonymous.
if(!$hasSoftCredit && ($payment->User->Email == 'Not provided' || $payment->User->Email == '')){
$payment->User = null;
}
}
catch(Exception){
// Anonymous donations don't have these elements present and will throw an exception.
$payment->User = null;
}
$payment->Created = DateTimeImmutable::createFromFormat('n/j/Y', trim($detailsRow->findElement(WebDriverBy::xpath('//td[preceding-sibling::th[normalize-space(.) = "Created"]]'))->getText()));
$payment->IsRecurring = sizeof($headerRow->findElements(WebDriverBy::xpath('//td[contains(., "Recurring")]'))) > 0;
$payment->Amount = floatval(preg_replace('/[^0-9\.]/', '', trim($detailsRow->findElement(WebDriverBy::xpath('//td[preceding-sibling::th[normalize-space(.) = "Total Amount"]]'))->getText())));
$payment->Fee = floatval(preg_replace('/[^0-9\.]/', '', trim($detailsRow->findElement(WebDriverBy::xpath('//td[preceding-sibling::th[normalize-space(.) = "Fee"]]'))->getText())));
$transactionId = (string)($detailsRow->findElement(WebDriverBy::xpath('//td[preceding-sibling::th[normalize-space(.) = "ID"]]'))->getText());
$transactionId = str_replace('View on FS', '', $transactionId);
$transactionId = str_replace('View on Finance', '', $transactionId);
$payment->TransactionId = trim($transactionId);
// We might also get a case where the donation is on behalf of a company match, but there's not really a way to distinguish that. Do a rough check.
// See donation 00b60a22-eafa-44cb-9850-54bef9763e8d
if($payment->User !== null && !$hasSoftCredit && preg_match('/\b(L\.?L\.?C\.?|Foundation|President|Fund|Charitable)\b/ius', $payment->User->Name ?? '')){
$payment->User = null;
}
// All set - create the payment.
try{
$payment->Create();
}
catch(Exceptions\PaymentExistsException){
// Payment already exists, just continue.
$log->Write('Donation already in database.');
continue;
}
// Does this payment create a new Patron in the Patrons Circle?
// If the user is *already* a Patron, then we just create the payment without further processing.
if(
(
$payment->IsRecurring
&&
$payment->Amount >= 10
&&
$payment->Created >= $lastMonth
)
||
(
!$payment->IsRecurring
&&
$payment->Amount >= 100
&&
$payment->Created >= $lastYear
)
){
// This payment is eligible for the Patrons Circle!
if($payment->User !== null){
// Are we already a patron?
if(!Db::QueryBool('
SELECT exists(
select *
from Patrons
where UserId = ?
and Ended is null
)
', [$payment->UserId])){
// Not a patron yet, add them to the Patrons Circle.
$patron = new Patron();
$patron->UserId = $payment->UserId;
$patron->User = $payment->User;
$patron->User->Payments = [$payment];
$patron->IsAnonymous = (trim($detailsRow->findElement(WebDriverBy::xpath('//td[preceding-sibling::th[normalize-space(.) = "Attribution"]]'))->getText()) == 'Private');
$patron->IsSubscribedToEmails = $patron->User !== null && $patron->User->Email !== null;
try{
$patron->AlternateName = trim($detailsRow->findElement(WebDriverBy::xpath('//td[preceding-sibling::th[normalize-space(.) = "Attribution Text"]]'))->getText());
}
catch(Exception){
}
$log->Write('Adding donor as patron ...');
$patron->Create();
}
elseif(!$payment->IsRecurring && !$payment->IsMatchingDonation){
// User is already a patron, but they made another non-recurring, non-matching donation.
// Send a thank-you email.
$log->Write('Adding donor as patron ...');
$patron->Create();
$log->Write('Sending thank you email to patron donor donating extra.');
$em = new Email();
$em->To = $payment->User->Email ?? '';
$em->ToName = $payment->User->Name ?? '';
$em->From = EDITOR_IN_CHIEF_EMAIL_ADDRESS;
$em->FromName = EDITOR_IN_CHIEF_NAME;
$em->Subject = 'Thank you for supporting Standard Ebooks!';
$em->Body = Template::EmailDonationThankYou();
$em->TextBody = Template::EmailDonationThankYouText();
$em->Send();
}
}
elseif(!$payment->IsRecurring && !$payment->IsMatchingDonation){
// User is already a patron, but they made another non-recurring, non-matching donation.
// Send a thank-you email.
// Fully-anonymous, non-recurring donation eligible for the Patrons Circle. We can't create a `Patron` or thank them, but we do notify the admins.
$patron = new Patron();
$patron->User = new User();
$log->Write('Sending thank you email to patron donor donating extra.');
$em = new Email();
$em->To = ADMIN_EMAIL_ADDRESS;
$em->From = ADMIN_EMAIL_ADDRESS;
$em->Subject = 'New Patrons Circle member';
$em->Body = Template::EmailAdminNewPatron(['patron' => $patron, 'payment' => $payment]);
$em->TextBody = Template::EmailAdminNewPatronText(['patron' => $patron, 'payment' => $payment]);;
$em->Send();
}
}
elseif($payment->User !== null){
// Payment amount is not eligible for the Patrons Circle; send a thank you email anyway, but only if this is a non-recurring donation, or if it's their very first recurring donation.
$previousPaymentCount = Db::QueryInt('
SELECT count(*)
from Payments
where UserId = ?
and IsRecurring = true
', [$payment->UserId]);
// We just added a payment to the system, so if this is their very first recurring payment, we expect the count to be exactly 1.
if(!$payment->IsRecurring || $previousPaymentCount == 1){
$log->Write('Sending thank you email to non-patron donor.');
$em = new Email();
$em->To = $payment->User->Email ?? '';
$em->ToName = $payment->User->Name ?? '';
@ -245,52 +290,14 @@ try{
$em->Send();
}
}
elseif(!$payment->IsRecurring && !$payment->IsMatchingDonation){
// Fully-anonymous, non-recurring donation eligible for the Patrons Circle. We can't create a `Patron` or thank them, but we do notify the admins.
$patron = new Patron();
$patron->User = new User();
$em = new Email();
$em->To = ADMIN_EMAIL_ADDRESS;
$em->From = ADMIN_EMAIL_ADDRESS;
$em->Subject = 'New Patrons Circle member';
$em->Body = Template::EmailAdminNewPatron(['patron' => $patron, 'payment' => $payment]);
$em->TextBody = Template::EmailAdminNewPatronText(['patron' => $patron, 'payment' => $payment]);;
$em->Send();
}
}
elseif($payment->User !== null){
// Payment amount is not eligible for the Patrons Circle; send a thank you email anyway, but only if this is a non-recurring donation, or if it's their very first recurring donation.
Db::Query('
DELETE
from PendingPayments
where TransactionId = ?
', [$pendingPayment->TransactionId]);
$previousPaymentCount = Db::QueryInt('
SELECT count(*)
from Payments
where UserId = ?
and IsRecurring = true
', [$payment->UserId]);
// We just added a payment to the system, so if this is their very first recurring payment, we expect the count to be exactly 1.
if(!$payment->IsRecurring || $previousPaymentCount == 1){
$log->Write('Sending thank you email to non-patron donor.');
$em = new Email();
$em->To = $payment->User->Email ?? '';
$em->ToName = $payment->User->Name ?? '';
$em->From = EDITOR_IN_CHIEF_EMAIL_ADDRESS;
$em->FromName = EDITOR_IN_CHIEF_NAME;
$em->Subject = 'Thank you for supporting Standard Ebooks!';
$em->Body = Template::EmailDonationThankYou();
$em->TextBody = Template::EmailDonationThankYouText();
$em->Send();
}
}
Db::Query('
DELETE
from PendingPayments
where TransactionId = ?
', [$pendingPayment->TransactionId]);
$log->Write('Donation processed.');
$log->Write('Donation processed.');
}
}
}
@ -308,5 +315,5 @@ catch(Exception $ex){
throw $ex;
}
finally{
$driver->quit();
$driver?->quit();
}