Created
October 31, 2023 07:27
-
-
Save eyJhb/209961084e87fe8466a860287b3cee93 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#! /usr/bin/env nix-shell | |
#! nix-shell -i perl -p perl nixUnstable.perl-bindings | |
# pipe using stdin, to this script, where each line has the format | |
# <algo> <hash> <name> | |
# sha256 0q6gwd9pqh41f6j5p9jl92aji4p8kx9inffpha93d7jc3ndsfq6q rematch2_2.1.2.tar.gz | |
use strict; | |
use Nix::Store; | |
foreach my $line ( <STDIN> ) { | |
chomp( $line ); | |
my @spl = split(' ', $line); | |
print makeFixedOutputPath(0, $spl[0], $spl[1], $spl[2]) . "\n"; | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env nix-shell | |
#!nix-shell -i bash -p nix-eval-jobs git | |
WORKERS=1 | |
BINARY_CACHE="file:///home/eyjhb/nix-cache-test/cache-dir" | |
STORE_CACHE="https://cache.nixos.org" | |
# current directory of this script | |
SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" | |
# Git | |
GIT_REVISION="5550a85a087c04ddcace7f892b0bdc9d8bb080c8" | |
TMP_PATH="/tmp/nix-cache-tmp-dir" | |
TMP_NIXPKGS="$TMP_PATH/nixpkgs" | |
TMP_STORE_PATHS="$TMP_PATH/store-paths" | |
# move into our tmp directory | |
mkdir $TMP_PATH | |
pushd $TMP_PATH | |
# clone nixpkgs into TMP_PATH | |
git clone https://github.com/NixOS/nixpkgs.git $TMP_NIXPKGS | |
pushd $TMP_NIXPKGS | |
git pull | |
git checkout $GIT_REVISION | |
GIT_SHORT_REV="$(git rev-parse --short $GIT_REVISION)" | |
GIT_REV_COUNT="$(git rev-list --count $GIT_REVISION)" | |
popd | |
TMP_NIXPKGS=$(nix store add-path $TMP_NIXPKGS) | |
# TMP_NIXPKGS="/nix/store/i333qx94x4mcqqm84miqrj02kg3hlpsz-nixpkgs" | |
# eval `nixos/release-combined.nix` using nix-eval-jobs | |
mkdir $TMP_STORE_PATHS | |
pushd $TMP_STORE_PATHS | |
# tmp files for store paths | |
TMP_CACHE_RELEASE_FILE="$TMP_STORE_PATHS/eval-cache.json" | |
TMP_CACHE_TARBALLS_FILE="$TMP_STORE_PATHS/tarballs-cache.json" | |
TMP_RELEASE_STORE_PATHS_DRV_FILE="$TMP_STORE_PATHS/store-paths-drv" | |
TMP_RELEASE_STORE_PATHS_FILE="$TMP_STORE_PATHS/store-paths" | |
TMP_TARBALLS_STORE_PATHS_FILE="$TMP_STORE_PATHS/tarballs-store-paths" | |
echo "Using file $TMP_CACHE_RELEASE_FILE outputs of evaluating $TMP_NIXPKGS/nixos/release-combined.nix" | |
nix-eval-jobs \ | |
--workers $WORKERS \ | |
--keep-going \ | |
--quiet \ | |
--force-recurse \ | |
"$TMP_NIXPKGS/nixos/release-combined.nix" \ | |
--arg nixpkgs '{ outPath = builtins.storePath '$TMP_NIXPKGS'; rev = "'$GIT_REVISION'"; revCount = '$GIT_REV_COUNT'; shortRev = "'$GIT_SHORT_REV'"; }' \ | |
--arg stableBranch true \ | |
--arg supportedPlatforms '[ "x86_64-linux"]' \ | |
--option extra-binary-caches 'https://hydra.nixos.org/' \ | |
--option system x86_64-linux \ | |
-I nixpkgs=$TMP_NIXPKGS \ | |
> $TMP_CACHE_RELEASE_FILE | |
echo "Resulting file $TMP_CACHE_RELEASE_FILE" | |
# get store paths for both drv and outputs to separate files | |
cat $TMP_CACHE_RELEASE_FILE | jq -r '.drvPath' | sed '/^null$/d' | sort | uniq > $TMP_RELEASE_STORE_PATHS_DRV_FILE | |
cat $TMP_CACHE_RELEASE_FILE | jq -r '.outputs.out' | sed '/^null$/d' | sort | uniq > $TMP_RELEASE_STORE_PATHS_FILE | |
# get tarballs from tarballs.json | |
nix-instantiate --eval --json --strict "$TMP_NIXPKGS/maintainers/scripts/find-tarballs.nix" --arg expr "import \"$TMP_NIXPKGS/maintainers/scripts/all-tarballs.nix\"" > $TMP_CACHE_TARBALLS_FILE | |
cat $TMP_CACHE_TARBALLS_FILE | jq -r '.[] | (if (.type | length) > 0 then .type else (.hash | split("(-|:)";""))[0] end) + " " + .hash + " " + .name' | sort | uniq | "$SCRIPTPATH/get-fixed-output-pathv2.pl" > $TMP_TARBALLS_STORE_PATHS_FILE | |
# nix copy them | |
# cat store-paths | xargs -I{} bash -c 'nix copy --from https://cache.nixos.org --to "$BINARY_CACHE" --derivation "{}^*" | true' | |
echo "Copying stage 1" | |
cat $TMP_RELEASE_STORE_PATHS_FILE | xargs -t -I{} nix copy --from https://cache.nixos.org --to "$BINARY_CACHE" --derivation '{}^*' | |
echo "Copying stage 2" | |
cat $TMP_RELEASE_STORE_PATHS_FILE | xargs -t -I{} nix copy --from https://cache.nixos.org --to "$BINARY_CACHE" '{}' | |
echo "Copying stage 3" | |
cat $TMP_RELEASE_STORE_PATHS_DRV_FILE | xargs -t -I{} nix copy --to "$BINARY_CACHE" '{}' | |
# cat $TMP_RELEASE_STORE_PATHS_DRV_FILE | xargs -I{} nix copy --from https://cache.nixos.org --to "$BINARY_CACHE" --derivation '{}^*' | |
echo "Copying stage 4" | |
cat $TMP_TARBALLS_STORE_PATHS_FILE | xargs -t -I{} nix copy --to "$BINARY_CACHE" '{}' |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment