Skip to content

Instantly share code, notes, and snippets.

@zkamvar

zkamvar/find-optional-output-hubs.sh

Created September 18, 2024 19:59
Show Gist options
  • Save zkamvar/4ebfd30e8a758b4df6a1a9100853f1d2 to your computer and use it in GitHub Desktop.
Save zkamvar/4ebfd30e8a758b4df6a1a9100853f1d2 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
find-optional-output-hubs.sh
#!/usr/bin/env bash
#
# This will search GitHub for hubs that use quantiles, CDF, or PMF output
# types and check if they use "optional" in their hub-config/tasks.json
# files.
echo "======== SEARCHING GITHUB ============"
curl -L \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer $(gh auth token)" \
-H "X-GitHub-Api-Version: 2022-11-28" \
"https://api.github.com/search/code?q=path:hub-config+quantile+language:json" \
| jq ".items[] | select(.name == \"tasks.json\") | {path: .url, repo: .repository.full_name}" \
| jq -s > quantile.json
curl -L \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer $(gh auth token)" \
-H "X-GitHub-Api-Version: 2022-11-28" \
"https://api.github.com/search/code?q=path:hub-config/+pmf+language:json" \
| jq ".items[] | select(.name == \"tasks.json\") | {path: .url, repo: .repository.full_name}" \
| jq -s > pmf.json
curl -L \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer $(gh auth token)" \
-H "X-GitHub-Api-Version: 2022-11-28" \
"https://api.github.com/search/code?q=path:hub-config/+cdf+language:json" \
| jq ".items[] | select(.name == \"tasks.json\") | {path: .url, repo: .repository.full_name}" \
| jq -s> cdf.json
echo "======== DOWNLOADING FILES ============"
echo ">>> QUANTILES"
thing=quantile
echo "| fields with optional ${thing} | repository |"
echo "| ----------------------------- | ---------- |"
for i in $(seq 0 $(( $(jq length quantile.json) - 1)));
do
repo=$(jq -r ".["$i"].repo" quantile.json)
path=$(jq -r ".["$i"].path" quantile.json)
if [[ -e "$repo" ]]; then
# echo "--- already downloaded: $repo"
:
else
echo "Downloading $repo ==================="
mkdir -p "$repo"
(
cd "$repo"
curl -L \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer $(gh auth token)" \
-H "X-GitHub-Api-Version: 2022-11-28" \
"$path" | jq -r .content | base64 --decode > tasks.json || echo "oops: $repo"
)
fi
n=$(jq "[.rounds[].model_tasks[].output_type.${thing}.output_type_id.optional | if . == null then 0 else 1 end] | add" "$repo/tasks.json")
if [[ $n == 0 ]]; then
:
else
echo "| $n | https://github.com/$repo |"
fi
done
echo ">>> CDF"
thing=cdf
echo "| fields with optional ${thing} | repository |"
echo "| ----------------------------- | ---------- |"
for i in $(seq 0 $(( $(jq length cdf.json) - 1)));
do
repo=$(jq -r ".["$i"].repo" cdf.json)
path=$(jq -r ".["$i"].path" cdf.json)
if [[ -e "$repo" ]]; then
# echo "--- already downloaded: $repo"
:
else
echo "Downloading $repo ==================="
mkdir -p "$repo"
(
cd "$repo"
curl -L \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer $(gh auth token)" \
-H "X-GitHub-Api-Version: 2022-11-28" \
"$path" | jq -r .content | base64 --decode > tasks.json || echo "oops: $repo"
)
fi
n=$(jq "[.rounds[].model_tasks[].output_type.${thing}.output_type_id.optional | if . == null then 0 else 1 end] | add" "$repo/tasks.json")
if [[ $n == 0 ]]; then
:
else
echo "| $n | https://github.com/$repo |"
fi
done
echo ">>> PMF"
thing=pmf
echo "| fields with optional ${thing} | repository |"
echo "| ----------------------------- | ---------- |"
for i in $(seq 0 $(( $(jq length pmf.json) - 1)));
do
repo=$(jq -r ".["$i"].repo" pmf.json)
path=$(jq -r ".["$i"].path" pmf.json)
if [[ -e "$repo" ]]; then
# echo "--- already downloaded: $repo"
:
else
echo "Downloading $repo ==================="
mkdir -p "$repo"
(
cd "$repo"
curl -L \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer $(gh auth token)" \
-H "X-GitHub-Api-Version: 2022-11-28" \
"$path" | jq -r .content | base64 --decode > tasks.json || echo "oops: $repo"
)
fi
n=$(jq "[.rounds[].model_tasks[].output_type.${thing}.output_type_id.optional | if . == null then 0 else 1 end] | add" "$repo/tasks.json")
if [[ $n == 0 ]]; then
:
else
echo "| $n | https://github.com/$repo |"
fi
done
@zkamvar
Copy link
Author

zkamvar commented Sep 18, 2024

as of 2024-09-18:

$ bash find-optional-output-hubs.sh 
======== SEARCHING GITHUB ============
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  159k  100  159k    0     0   306k      0 --:--:-- --:--:-- --:--:--  306k
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 23947  100 23947    0     0   102k      0 --:--:-- --:--:-- --:--:--  102k
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 42538  100 42538    0     0   155k      0 --:--:-- --:--:-- --:--:--  155k
======== DOWNLOADING FILES ============
>>> QUANTILES
fields with optional quantile repository
1 https://github.com/midas-network/covid19-scenario-modeling-hub
4 https://github.com/Testing-Forecast-Actions/ScenarioModellingHub
2 https://github.com/micokoch/simple-hub
13 https://github.com/midas-network/example_round-scenariohub
3 https://github.com/european-modelling-hubs/RespiCompass
2 https://github.com/midas-network/rsv-scenario-modeling-hub
6 https://github.com/midas-network/flu-scenario-modeling-hub
1 https://github.com/Testing-Forecast-Actions/TestingValidations
1 https://github.com/hubverse-org/example-complex-scenario-hub
4 https://github.com/midas-network/covid19-smh-research
2 https://github.com/LucieContamin/example_smh 
>>> CDF
fields with optional cdf repository
2 https://github.com/hubverse-org/flusight_hub_archive 
>>> PMF
fields with optional pmf repository
1 https://github.com/hubverse-org/flusight_hub_archive

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment