Clean up sync-ebooks and add --token option.

scripts/README.md

@@ -1,6 +1,6 @@
 # sync-ebooks
 
-To use, call this script with the directory where your ebooks go as its last argument. For example `sync-ebooks /standardebooks.org/ebooks` or if you want to clone them into this repository `sync-ebooks ebooks`. If you want progress output, use -v, and if you want detailed git progress output use -vv.
+To use, call this script with the directory where your ebooks go as its last argument. For example `sync-ebooks /standardebooks.org/ebooks` or if you want to clone them into this repository `sync-ebooks ebooks`. If you want progress output, use -v, and if you want detailed git progress output use -vv. GitHub allows you to make 60 unauthenticated API requests per hour. If you use unauthenticated API requests for other things, this might not be enough, so to resolve that issue, you can create a new OAuth token at https://github.com/settings/tokens/new and pass it via the --token option. You don't need to give the token any permissions.
 
 # deploy-ebook-to-www
 

scripts/sync-ebooks

@@ -8,11 +8,12 @@ DESCRIPTION
 	Syncs books from standardebooks.org GitHub org to specified folder.
 
 USAGE
-	${0##*/} [-v,-vv,--verbosity=INTEGER] [-u,--update-only] DIRECTORY
+	${0##*/} [-v,-vv,--verbosity INTEGER] [-u,--update-only] [--token TOKEN] DIRECTORY
 
-	With -v or --verbosity=1, display general progress updates.
+	With -v or --verbosity 1, display general progress updates.
-	With -vv or --verbosity=2, display general progress updates and verbose git output.
+	With -vv or --verbosity 2, display general progress updates and verbose git output.
 	With --update-only, only sync existing repositories, do not download new repositories.
+	With --token TOKEN, specify a GitHub access token to use for request. Useful for when you hit the rate limit.
 
 	DIRECTORY should be where the repositories should go.
 
@@ -22,51 +23,63 @@ EOF
 	exit
 }
 die(){ printf "\033[0;7;31mError:\033[0m %s\n" "${1}" 1>&2; exit 1; }
-if [ $#  -eq 1 ]; then if [ "$1" = "--help" ] || [ "$1" = "-h" ]; then usage; fi fi
+require(){ command -v "$1" > /dev/null 2>&1 || { suggestion=""; if [ -n "$2" ]; then suggestion=" $2"; fi; die "$1 is not installed.${suggestion}"; } }
 # End boilerplate
 
+require "git" "Try: apt-get install git"
+
 # Terminate on CTRL-C
 trap ctrl_c INT
 ctrl_c() {
 	exit
 }
 
-if [[ $# -lt 1 ]]; then
+if [[ $# -eq 0 ]]; then
 	usage
 fi
 
 verbosity=0
 updateOnly="false"
+githubToken=""
 target=""
 
-for i in "$@"
+while [ $# -gt 0 ]; do
-do
+	case "$1" in
-	case $i in
 		-h|--help)
-			usage
+			usage ;;
-		;;
 		-v)
 			verbosity=1
-			shift
+			shift 1
-		;;
+			;;
 		-vv)
 			verbosity=2
-			shift
+			shift 1
-		;;
+			;;
-		--verbosity=*)
+		--verbosity)
-			verbosity=${i//[-a-zA-Z0-9]*=/}
+			case "$2" in
-			shift
+				''|*[!0-9]*) die "Verbosity is not an integer." ;;
-		;;
+			esac
-		-u|--update-only)
+			verbosity="$2"
-			updateOnly="true"
+			shift 2
-			shift
+			;;
-		;;
+		--token)
+			case "$2" in
+				''|*[!0-9a-zA-Z]*) die "Token is empty or contains illegal characters." ;;
+			esac
+			githubToken="$2"
+			shift 2
+			;;
 		*)
-			target="${i}"
+			break ;;
-			shift
 	esac
 done
 
+if [ $# -ne 1 ] || [ -z "$1" ]; then
+	usage
+fi
+
+target="$1"
+
 if ! [ -d "${target}" ]; then
 	die "${target} is not a directory."
 fi
@@ -82,14 +95,19 @@ fi
 for item in ./*; do
 	[ -e "${item}" ] || break
 
-	if [ "${verbosity}" -eq 1 ]; then
+	if [ "${verbosity}" -gt 0 ]; then
 		printf "Updating %s ... " "${item}"
+	fi
+
+	if [ "${verbosity}" -lt 2 ]; then
 		git -C "${item}" fetch -q
-		printf "Done.\n"
+	else
-	elif [ "${verbosity}" -gt 1 ]; then
-		printf "Updating %s ... \n" "${item}"
 		git -C "${item}" fetch -v
 	fi
+
+	if [ "${verbosity}" -gt 0 ]; then
+		printf "Done.\n"
+	fi
 done
 
 if [ "${updateOnly}" = "true" ]; then
@@ -101,14 +119,32 @@ if [ "${verbosity}" -gt 0 ]; then
 	printf "Fetching repository urls ..."
 fi
 
-page=1
+url="https://api.github.com/orgs/standardebooks/repos?per_page=100"
-urls=""
+repoUrls=""
-pageurls="placeholder"
+
-while [ -n "${pageurls}" ]; do
+while true; do
-	pageurls=$(curl -s "https://api.github.com/orgs/standardebooks/repos?per_page=100&page=${page}" \
+	if [ -n "${githubToken}" ]; then
-		| awk 'BEGIN { FS="\""; RS="," }; { if ($2 == "clone_url") {print $4} }')
+		response=$(curl -H "Authorization: token ${githubToken}" -si "${url}") ||
-	urls=$(printf "%s\n%s" "${urls}" "${pageurls}")
+			die "Curl request failed."
-	page=$((page + 1))
+	else
+		response=$(curl -si "${url}") ||
+			die "Curl request failed."
+	fi
+
+	if printf "%s" "${response}" | grep -q "^X-RateLimit-Remaining: 0$"; then
+		limitReset=$(printf "%s" "${response}" | grep -oP "^X-RateLimit-Reset: \K[0-9]+$")
+		printf "You have reached your daily allowance for unauthenticated GitHub API requests.\n\
+			Either wait until %s or use an OAuth token.\n\
+			You can create a new token at https://github.com/settings/tokens/new and \
+			pass it to this script with the --token option.\n\
+			The token does not need any permissions.\n" "$(date -d @"${limitReset}")" 1>&2
+		exit
+	fi
+
+
+	currentRepoUrls=$(printf "%s" "${response}" | awk 'BEGIN { FS="\""; RS="," }; { if ($2 == "clone_url") {print $4} }')
+	repoUrls=$(printf "%s\n%s" "${repoUrls}" "${currentRepoUrls}")
+	url=$(printf "%s" "${response}" | grep -oP "<\Khttps://api.github.com/[^>]*(?=>; rel=\"next\",)") || break
 
 	if [ "${verbosity}" -gt 0 ]; then
 		printf "."
@@ -119,29 +155,25 @@ if [ "${verbosity}" -gt 0 ]; then
 	printf " Done.\n"
 fi
 
-urls=$(printf "%s\n" "${urls}" | grep -v -e "/tools.git\$" -e "/web.git\$" -e "/manual.git\$" | awk 'NF')
+repoUrls=$(printf "%s" "${repoUrls}" | grep -v -e "/tools.git\$" -e "/web.git\$" -e "/manual.git\$" | awk 'NF')
 
-printf "%s\n" "${urls}" | while IFS= read -r repourl; do
+printf "%s\n" "${repoUrls}" | while IFS= read -r repoUrl; do
-	[ -n "${repourl}" ] || continue
+	[ -n "${repoUrl}" ] || continue
-	[ -d "${repourl##*/}" ] && continue
+	[ -d "${repoUrl##*/}" ] && continue
 
-	if [ "${verbosity}" -gt 1 ]; then
+	if [ "${verbosity}" -gt 0 ]; then
-		printf "Cloning %s ... \n" "${repourl}"
+		printf "Cloning %s ... \n" "${repoUrl}"
-		git clone -v --bare "${repourl}"
+	fi
-		if ! [ -d "${repourl##*/}" ]; then
+
-			printf "Failed.\n"
+	if [ "${verbosity}" -lt 2 ]; then
-		fi
+		git clone -q --bare "${repoUrl}"
 	else
-		if [ "${verbosity}" -gt 0 ]; then
+		git clone -v --bare "${repoUrl}"
-			printf "Cloning %s ... " "${repourl}"
+	fi
-		fi
+
-		git clone -q --bare "${repourl}"
+	if ! [ -d "${repoUrl##*/}" ]; then
-		if [ "${verbosity}" -gt 0 ]; then
+		printf "Failed to clone %s.\n" "${repoUrl##*/}." 1>&2
-			if ! [ -d "${repourl##*/}" ]; then
+	elif [ "${verbosity}" -gt 0 ]; then
-				printf "Failed.\n"
+		printf "Done.\n"
-			else
-				printf "Done.\n"
-			fi
-		fi
 	fi
 done