mpadge.tngl.sh
dot dot dotfiles
1#!/bin/bash
2
3# GitHub has started restricting daily cache usage to 10GB, but only allows
4# cache usage to be checked for individual repositories, with no way of getting
5# an overview of which repos are using excessive cache.
6#
7# This script runs `gh cache` command on aslected sub-directories of specified
8# paths, filters results with `jq` to previous 24-hour period, and puts all
9# combined results in to a single output file in the directory where this
10# script is run.
11#
12# The script should be run in a directory presumed to be two directories below
13# where all git repositories sit, so each sub-dir should be `<org>/<repo>`.
14
15if ! command -v jq &>/dev/null; then
16 echo "jq must be installed."
17 exit 1
18fi
19if ! command -v gh &>/dev/null; then
20 echo "gh must be installed."
21 exit 1
22fi
23
24json_file="gh-cache-dat.json" # files stored in each directory from `gh cache`
25output_file="combined_data.json" # final output stored in pwd
26
27echo "=== Running 'gh cache' on sub-directories ==="
28find . -mindepth 2 -maxdepth 2 -type d | \
29 grep -e "mpadge" -e "ropensci/" -e "UrbanAnalyst" -e "ropensci-review-tools" \
30 | while read -r dir; do
31
32 cd "$dir" || continue
33
34 if gh cache list --json createdAt,sizeInBytes >> "$json_file" 2>/dev/null; then
35 echo " Generated dat.json in $dir successfully"
36 if [ -f "$json_file" ] && [ -s "$json_file" ]; then
37 twenty_four_hours_ago=$(date -u -d '24 hours ago' +'%Y-%m-%dT%H:%M:%SZ')
38 jq --arg cutoff "$twenty_four_hours_ago" '
39 if type == "array" then
40 [.[] | select(.createdAt >= $cutoff)]
41 else
42 select(.createdAt >= $cutoff)
43 end
44 ' $json_file > temp.json && mv temp.json $json_file
45 fi
46 else
47 echo " No gh cache from $dir"
48 fi
49
50 cd - > /dev/null
51done
52
53echo "=== Combining data ==="
54temp_file="combined_data_temp.json"
55dirname=$(pwd)
56
57echo "[" > "$temp_file"
58first_entry=true
59
60find . -mindepth 2 -maxdepth 3 -name "$json_file" | \
61 grep -e "mpadge" -e "ropensci/" -e "UrbanAnalyst" -e "ropensci-review-tools" \
62 | while read -r json_file; do
63
64 json_path="${dirname}/${json_file#./}"
65
66 if [ -s "$json_path" ] && command -v jq &>/dev/null; then
67 json_content=$(cat "$json_file")
68 if [ "$json_content" != "[]" ] && [ "$json_content" != "null" ] && [ -n "$json_content" ]; then
69 if [ "$first_entry" = false ]; then
70 echo "," >> "$temp_file"
71 fi
72
73 echo " jq on ${json_path} ..."
74 jq --arg path "$json_path" '
75 if type == "array" then
76 [.[] | .path = $path]
77 else
78 .path = $path
79 end
80 ' "$json_path" | sed '1d; $d' >> "$temp_file"
81
82 first_entry=false
83 fi
84 fi
85 rm "$json_path"
86done
87
88echo "]" >> "$temp_file"
89
90# Then group results by "path" and store total sum of cache sizes:
91if command -v jq &>/dev/null; then
92 jq 'group_by(.path) |
93 map({
94 path: .[0].path,
95 total_size_mb: (((map(.sizeInBytes) | add) / (1024 * 1024)) | round),
96 count: length
97 })' "$temp_file" > "$output_file"
98else
99 mv "$temp_file" "$output_file"
100fi
101
102rm -f "$temp_file"
103
104echo "Combined data saved to $output_file"
105echo "Total MB of cache in previous 24 hours:"
106jq . "$output_file"
107total_mb="$(jq 'map(.total_size_mb) | add' $output_file)"
108
109echo "Overall total: $total_mb MB"