Skip to content

Instantly share code, notes, and snippets.

@shiryel

shiryel/nix_try_eval.patch

Last active May 29, 2024 22:51
Show Gist options
  • Save shiryel/16d3e98dcef2998316692f2071f29b9f to your computer and use it in GitHub Desktop.
Save shiryel/16d3e98dcef2998316692f2071f29b9f to your computer and use it in GitHub Desktop.
Download ZIP
Nixpkgs to JSON
Raw
nix_try_eval.patch
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index bc253d0a3..e6c348d99 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -867,7 +867,7 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Va
state.forceValue(*args[0], pos);
attrs.insert(state.sValue, args[0]);
attrs.alloc("success").mkBool(true);
- } catch (AssertionError & e) {
+ } catch (EvalError & e) {
attrs.alloc(state.sValue).mkBool(false);
attrs.alloc("success").mkBool(false);
}
Raw
nixpkgs_to_json_v1.nix
{
description = "Nixpkgs to JSON";
inputs = {
nixpkgs_stable.url = "github:NixOS/nixpkgs/nixos-22.05";
flake-utils.url = "github:numtide/flake-utils/v1.0.0";
};
outputs = { self, nixpkgs_stable, flake-utils }:
with builtins;
with nixpkgs_stable.lib;
let
params = import ./params.nix;
pkgs = import nixpkgs_stable {
system = "x86_64-linux";
config.android_sdk.accept_license = true;
# config.allowUnfree = true;
};
in
rec {
attrCount = x: pipe x [
(mapAttrsToList nameValuePair)
length
];
getMeta = attr: acc:
if isAttrs attr then
# some attr.meta can be a boolean (wtf)
if attr ? meta && isAttrs attr.meta then
let meta = attr.meta;
in
{
path = acc;
name = if attr ? name then attr.name else lists.last acc;
meta = (meta
//
# sometimes it has a list of lists of platforms
(if meta ? platforms then
{ platforms = lists.flatten meta.platforms; }
else
{ })
//
# ignore maintainers that do not have email
# like `["roblabla"]`
(if meta ? maintainers && meta.maintainers ? name then
{ }
else
{ maintainers = [ ]; }
)
//
# normalize .homepage to list
(if meta ? homepage && !isList meta.homepage then
{ homepage = [ meta.homepage ]; }
else
{ }
)
);
}
else
# remove the attr, transforming it on a list of path/meta
pipe attr [
(mapAttrsToList
(name: value:
# test if its possible to evaluate
# ((match "androidndkPkgs.*" name) == null) && ((debug.traceVal name) != false) && (tryEval (typeOf value == "set")).value
# NOTE: cuda is cursed ¯\_(ツ)_/¯
if (lists.all (x: x != name) (acc ++ [ "cuda" ]))
&& (tryEval (isAttrs value)).value
&& (attrCount value) < 200
then
pipe (getMeta value (acc ++ [ name ])) [
# flatten packages inside packages
lists.flatten
]
else
[ ]
))
]
else
[ ];
attrToJSON = pipe pkgs [
(mapAttrsToList nameValuePair)
(x: lists.sublist params.start params.count x)
listToAttrs
(x: getMeta x [ "pkgs" ])
# flatten packages inside packages
lists.flatten
toJSON
];
};
}
Raw
nixpkgs_to_json_v2.nix
{
description = "Nixpkgs to JSON";
inputs = {
nixpkgs_stable.url = "github:NixOS/nixpkgs/nixos-22.11";
nixpkgs_unstable.url = "github:NixOS/nixpkgs/nixos-unstable";
flake-utils.url = "github:numtide/flake-utils/v1.0.0";
};
outputs = { self, nixpkgs_stable, nixpkgs_unstable, flake-utils }:
with builtins;
with nixpkgs_stable.lib;
let
params = import ./params.nix;
config = {
android_sdk.accept_license = true;
allowUnfree = true;
allowBroken = true;
allowUnsupportedSystem = true;
permittedInsecurePackages = true;
input-fonts.acceptLicense = true;
joypixels.acceptLicense = true;
};
pkgs_stable = import nixpkgs_stable {
system = "x86_64-linux";
config = config;
};
pkgs_unstable = import nixpkgs_unstable {
system = "x86_64-linux";
config = config;
};
pkgs = if params.version == "unstable" then pkgs_unstable else pkgs_stable;
ignore_list = [
# not necessary to evaluate
#"drvAttrs"
#"inputDerivation"
"lib" # functions
"pkgsCross" # cross-compiled packages
"config" # options
"nixosTests" # nixpkgs tests
"tests" # functions tests
"stdenv" # it has meta and a lot of packages have stdenv internally
];
# how much packages per batch
# can cause stack overflows!
offset = params.offset;
size = params.size;
in
rec {
#
# API
#
get_pkgs = pkgs_unstable;
# divide and conquer for `findSuperPackagesPath`
getSuperPackagesPath =
let
attr_path = getAttr params.attr_path pkgs;
in
pipe attr_path [
#(x: (debug.traceVal x))
(mapAttrsToList nameValuePair)
(lists.sublist (offset * size) size)
listToAttrs
(x: findSuperPackagesPath x (params.attr_path))
lists.flatten
toJSON
];
# divide and conquer for `findPackagesMeta`
getPackagesMeta =
let
attr_path = getAttr params.attr_path pkgs;
in
pipe attr_path [
(mapAttrsToList nameValuePair)
(lists.sublist (offset * size) size)
listToAttrs
(x: findPackagesMeta x (params.attr_path))
lists.flatten
toJSON
];
#
# Support
#
# counts how much attributes a pkg node has
# -> attrCount {...pkgs...}
attrCount = x: pipe x [
(mapAttrsToList nameValuePair)
length
];
# traverses the pkgs nodes using a list of names
# -> getAttr [ "pkgs" "ArchiSteamFarm" ] {...pkgs...}
getAttr = list: attr:
lists.last
(foldl'
(acc: x: (catAttrs x acc))
[ attr ] # current pkg node
list # attr_path list
);
# GENERIC FUNCTION
# traverses the pkgs nodes until it find the meta attr
travelPackages = attr: acc: fn_found: fn_else:
if isAttrs attr && !(isOption attr) then
if attr ? meta
# some attr.meta can be a boolean or string
# and to avoid cases where the meta attrs can't be
# evaluated we only get what has platforms
&& (tryEval (isAttrs attr.meta)).value
&& (tryEval (attr.meta.platforms)).success then
fn_found attr
else
pipe attr [
(mapAttrsToList
(name: value:
if (lists.all (x: x != name) (acc ++ ignore_list))
#&& ((debug.traceVal (acc ++ [ name ])) != false)
#
# if it got rejected by the first if and has meta attr
# then its completely broken!!! (and highly explosive)
&& !(tryEval (isAttrs attr.meta)).value
&& (tryEval (isAttrs value)).value # only attrs
&& !(hasAttr name value) # ignore if recursive attr
then
fn_else name value
else
[ ]
))
]
else
[ ];
normalizeMeta = meta: (
meta
//
# sometimes it has a list of lists of platforms
{ platforms = lists.flatten meta.platforms; }
//
# ignore maintainers that do not have email
# like `["roblabla"]`
(if meta ? maintainers && (lists.any isString meta.maintainers) then
{ maintainers = [ ]; }
else
{ }
)
//
# normalize .homepage to list
(if meta ? homepage && !isList meta.homepage then
{ homepage = [ meta.homepage ]; }
else
{ }
)
//
(if meta ? license then
{
licenses = pipe meta [
(m:
(if isList m.license then m.license else [ m.license ])
++ (if m ? licenses then m.licenses else [ ])
)
lists.flatten
(map (x: (
if isString x then { shortName = x; } else x
)
))
];
}
else
{ }
)
);
#
# Traversers
#
# find pkgs nodes that have more than 200 attributes
# returning results like: {"r":["pkgs","ArchiSteamFarm"]}
# to be used with findPackagesMeta
findSuperPackagesPath = attr: acc:
travelPackages attr acc
# fn_found
(attr: [ ])
# fn_else
(name: value:
if (attrCount value) > size then
{ r = acc ++ [ name ]; }
else
pipe value [
(x: findSuperPackagesPath x (acc ++ [ name ]))
lists.flatten
]
);
# find pkgs nodes with meta attributes
# using `normalizeMeta` on the pkgs metadata
findPackagesMeta = attr: acc:
travelPackages attr acc
# fn_found
(attr:
{
path = acc;
version =
if (tryEval attr.version).success
&& attr.version != "" then
attr.version
else
if (tryEval attr.meta.version).success
&& attr.meta.version != "" then
attr.meta.version
else
if (tryEval attr.meta.name).success
&& attr.meta.name != "" then
attr.meta.name
else
lists.last acc;
meta =
if (tryEval attr.meta).success then
normalizeMeta attr.meta
else
{ };
}
)
# fn_else
(name: value:
if (attrCount value) > size then
[ ]
else
pipe value [
(x: findPackagesMeta x (acc ++ [ name ]))
lists.flatten
]
);
};
}
Raw
nixpkgs_to_json_v3.nix
# nix-eval-jobs --meta --flake path:priv/nix_eval2#get
{
description = "Nixpkgs to JSON";
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.11";
};
outputs = { self, nixpkgs, }:
with builtins;
with nixpkgs.lib;
let
config = {
android_sdk.accept_license = true;
allowUnfree = true;
allowBroken = true;
allowUnsupportedSystem = true;
permittedInsecurePackages = true;
input-fonts.acceptLicense = true;
joypixels.acceptLicense = true;
segger-jlink.acceptLicense = true;
};
pkgs = import nixpkgs {
system = "x86_64-linux";
config = config;
};
ignore_list = [
"lib" # functions
"pkgsCross" # cross-compiled packages
"config" # options
"nixosTests" # nixpkgs tests
"tests" # functions tests
"stdenv" # it has meta and a lot of packages have stdenv internally
"iosSdkPkgs"
];
in
rec {
#
# API
#
get = pkgs;
getSize = pipe pkgs [
(mapAttrsToList nameValuePair)
length
toJSON
];
getMidSize = { size ? 100, offset ? 0 }:
pipe
pkgs
[
(mapAttrsToList nameValuePair)
(lists.sublist (offset * size) size)
listToAttrs
(x: findPackagesMeta x [ ])
lists.flatten
length
# WORKAROUND: Nix does not accept `--arg` on flakes[1], so we use
# nix-eval-jobs, and them we get the size using a derivation name
# [1] - https://github.com/NixOS/nix/issues/3843#issuecomment-661710951
(x: derivation {
name = toJSON x;
builder = ":";
system = "x86_64-linux";
})
];
getSlice = { size ? 100, offset ? 0, mid_size ? 1, mid_offset ? 0 }:
pipe
pkgs
[
(mapAttrsToList nameValuePair)
# WORKAROUND: Nix can't evaluate every package, even with
# nix-eval-jobs, so we evaluate one by one :)
(lists.sublist (offset * size) size)
listToAttrs
(x: findPackagesMeta x [ ])
lists.flatten
# WORKAROUND: Nix can't handle big amounts of data, and some
# grouped packages are too big for it, so we need to split the split
(lists.sublist (mid_offset * mid_size) mid_size)
(foldl (attrsets.recursiveUpdate) { })
];
#
# Support
#
# GENERIC FUNCTION
# traverses the pkgs nodes until it find the meta attr
travelPackages = attr: acc: fn_found: fn_else:
if isAttrs attr && !(isOption attr) then
if attrsets.isDerivation attr
#&& ((debug.traceVal acc) != false)
then
#{name = acc; value = attr;}
fn_found (attrsets.setAttrByPath acc attr)
else
pipe attr [
(mapAttrsToList
(name: value:
if (lists.all (x: x != name) (acc ++ ignore_list))
&& !((length acc) > 1)
&& !(strings.hasPrefix "androidndkPkgs" name)
#&& ((debug.traceVal (acc ++ [ name ])) != false)
#
# if it got rejected by the first if and has meta attr
# then its completely broken!!! (and highly explosive)
#&& !(tryEval (attr ? meta && isAttrs attr.meta)).value
&& (tryEval (isAttrs value)).value # only attrs
#&& ((debug.traceVal (acc ++ [ name ])) != false)
&& !(tryEval (hasAttr name value)).value # ignore if recursive attr
#&& ((debug.traceVal (acc ++ [ name ])) != false)
then
fn_else name value
else
[ ]
))
]
else
[ ];
#
# Traversers
#
# find pkgs nodes with meta attributes
# using `normalizeMeta` on the pkgs metadata
findPackagesMeta = attr: acc:
travelPackages attr acc
# fn_found
(attr: attr)
# fn_else
(name: value:
pipe value [
(x: findPackagesMeta x (acc ++ [ name ]))
lists.flatten
]
);
};
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment