123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191 | #!/bin/bash
# Usage: tools/migrate-from-github [--dry-run] [--yes] [--owner OWNER]
# [--config FILE] [--key FILE] [--git-host HOST] SERVER
#
# Lists your GitHub repositories with gh and brings SERVER in line with
# them, for a server set up as in docs/openbsd.md:
#
# - a repository SERVER lacks is offered (or, with --yes, taken): it is
# created with new-repo, with GitHub's default branch and description
# and GitHub as its mirror, then filled from GitHub, every branch and
# tag, by a push that the hook renders
# - a repository SERVER has is wired up again with new-repo, which
# installs the current hook and records the config, and gets GitHub as
# a mirror if it lacks it, and GitHub's description if it has none
#
# Archived repositories are not mirrored, as GitHub refuses pushes to
# them, and empty ones are skipped. Everything that needs root runs in one
# doas on SERVER, after asking; --dry-run prints it instead, and the
# pushes, and changes nothing.
#
# SERVER what ssh logs you in to SERVER as, with doas
# --owner OWNER whose repositories (default: yours)
# --config FILE the renderer's config on SERVER
# (default /etc/repo-render/repo-render.yaml)
# --key FILE an ssh key on SERVER for pushing to GitHub, which each
# mirrored repository's core.sshCommand is set to use
# --git-host HOST where to push as git (default: SERVER's host name)
#
# Needs gh (logged in), mikefarah's yq, git and ssh here, and
# repo-render-new-repo on SERVER.
# shellcheck disable=SC2029 # the commands are meant to expand here, then run there
set -euo pipefail
usage() {
sed -n '2,3s/^# //p' "$0" >&2
exit 2
}
die() { echo "migrate-from-github: $*" >&2; exit 1; }
config=/etc/repo-render/repo-render.yaml
dry="" yes="" owner="" key="" git_host="" server=""
while (($#)); do
case $1 in
--dry-run) dry=1 ;;
--yes) yes=1 ;;
--owner) owner=${2?}; shift ;;
--config) config=${2?}; shift ;;
--key) key=${2?}; shift ;;
--git-host) git_host=${2?}; shift ;;
-*) usage ;;
*) [[ $server ]] && usage; server=$1 ;;
esac
shift
done
[[ $server ]] || usage
[[ $git_host ]] || git_host=${server#*@}
for p in gh yq git ssh; do
command -v "$p" >/dev/null || die "needs $p"
done
# These go into commands run as root on SERVER.
for p in "$config" "$key"; do
[[ -z $p || $p =~ ^/[A-Za-z0-9._/-]+$ ]] || die "$p: an absolute path of letters, digits and . _ - / only"
done
# Runs a command, or with --dry-run prints it, quoted to paste into a shell
run() { if [[ $dry ]]; then printf '%q ' "$@"; echo; else "$@"; fi; }
# Quotes a string for sh, in single quotes
sq() { printf "'%s'" "${1//\'/\'\\\'\'}"; }
# A command line for sh, each word quoted
sqline() { local out="" w; for w in "$@"; do out+="${out:+ }$(sq "$w")"; done; printf '%s' "$out"; }
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT
git_dir=$(ssh "$server" cat "$config" | yq '.git_dir // ""')
[[ $git_dir =~ ^/[A-Za-z0-9._/-]+$ ]] || die "$config on $server has no usable git_dir (got '$git_dir')"
# Each repository on SERVER: its description, mirrors and config, read
# from its files, which needs no git as their owner.
declare -A on_server descs mirrors wired sshcmd
while IFS=$'\x1f' read -r name desc ms cfg sc; do
[[ $desc == "Unnamed repository"* ]] && desc=""
on_server[$name]=1 descs[$name]=$desc mirrors[$name]=" $ms " wired[$name]=$cfg sshcmd[$name]=$sc
done < <(ssh "$server" "cd $(sq "$git_dir") && for d in *.git; do [ -f \"\$d/config\" ] || continue
c=\"\$d/config\"
printf '%s\\037%s\\037%s\\037%s\\037%s\\n' \"\${d%.git}\" \"\$(head -n 1 \"\$d/description\" 2>/dev/null)\" \
\"\$(git config -f \"\$c\" --get-all repo-render.mirror | tr '\\n' ' ')\" \
\"\$(git config -f \"\$c\" repo-render.config)\" \"\$(git config -f \"\$c\" core.sshCommand)\"
done; true")
key_cmd=""
[[ $key ]] && key_cmd="ssh -o BatchMode=yes -o StrictHostKeyChecking=yes -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -i $key"
as_git=() # commands to run as git on SERVER, each a quoted line
fill=() # "name url": new repositories to fill from GitHub
render=() # existing repositories whose description changed
list=(gh repo list --limit 1000 --json "name,nameWithOwner,defaultBranchRef,isArchived,isFork,description" \
--jq '.[] | [.name, .nameWithOwner, (.defaultBranchRef.name // ""), (.isArchived | tostring), (.isFork | tostring), ((.description // "") | gsub("[\\n\\r\\t]"; " "))] | join("\u001f")')
[[ $owner ]] && list=("${list[@]:0:3}" "$owner" "${list[@]:3}")
# The list comes in on fd 3, so the questions read the terminal. Fields
# are split on \x1f: bash merges adjacent tabs, losing an empty field.
while IFS=$'\x1f' read -r -u 3 name full branch archived fork description; do
if [[ ! $name =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ || $name == *.git ]]; then
echo "$full: skipped, the name is not one the renderer takes"
continue
fi
url=git@github.com:$full.git
dir=$git_dir/$name.git
args=() status=()
mirror=1
if [[ $archived == true ]]; then
mirror="" status+=("archived on GitHub, so not mirrored")
fi
if [[ ! -v on_server[$name] ]]; then
if [[ ! $branch ]]; then
echo "$name: empty on GitHub, skipped"
continue
fi
[[ $branch =~ ^[A-Za-z0-9_][A-Za-z0-9._/-]*$ ]] || { echo "$name: skipped, default branch '$branch' is not one the renderer takes"; continue; }
what="$name: not on $server"
[[ $fork == true ]] && what+=" (a fork)"
if [[ ! $yes ]]; then
read -rp "$what. Put it there? [y/N] " answer
[[ $answer == [yY]* ]] || continue
fi
args+=(-c "$config" -b "$branch")
[[ $description ]] && args+=(-d "$description")
[[ $mirror ]] && args+=(-m "$url")
fill+=("$name $url")
status=("to create" "${status[@]}")
else
if [[ ${wired[$name]} != "$config" ]]; then
args+=(-c "$config")
status+=("to wire up")
fi
if [[ $mirror && ${mirrors[$name]} != *" $url "* ]]; then
args+=(-m "$url")
status+=("mirror to add")
fi
if [[ $description && ! ${descs[$name]} ]]; then
args+=(-d "$description")
render+=("$name")
status+=("description to add")
fi
fi
if ((${#args[@]})); then
as_git+=("$(sqline repo-render-new-repo "${args[@]}" "$dir")")
fi
if [[ $key_cmd && $mirror && ${sshcmd[$name]-} != "$key_cmd" ]]; then
as_git+=("$(sqline git -C "$dir" config core.sshCommand "$key_cmd")")
status+=("mirror key to set")
fi
((${#status[@]})) || status=("up to date")
line="$name: ${status[0]}"
for st in "${status[@]:1}"; do line+="; $st"; done
echo "$line"
done 3< <("${list[@]}")
for name in "${render[@]}"; do
# The description is on every page, so they are all redone.
as_git+=("$(sqline repo-render --config "$config" render "$name")")
done
if ((${#as_git[@]})); then
root=""
for c in "${as_git[@]}"; do
root+="su -s /bin/sh git -c $(sq "$c"); "
done
if [[ $dry ]]; then
echo "ssh -t $server doas sh -ec, running as git:"
printf ' %s\n' "${as_git[@]}"
else
echo "To run as git on $server:"
printf ' %s\n' "${as_git[@]}"
if [[ ! $yes ]]; then
read -rp "Run these? [y/N] " answer
[[ $answer == [yY]* ]] || { echo "nothing changed"; exit 1; }
fi
ssh -t "$server" "doas sh -ec $(sq "$root")"
fi
fi
# Fill each new repository from GitHub: every branch, and the tags. The
# push runs the hook, which renders it.
for f in "${fill[@]}"; do
read -r name url <<<"$f"
run git clone --quiet --bare "$url" "$tmp/$name.git"
run git -C "$tmp/$name.git" push --quiet --all "git@$git_host:$name.git"
run git -C "$tmp/$name.git" push --quiet --tags "git@$git_host:$name.git"
done
|