|
require('dev') |
|
require('set') |
|
require('English') |
|
|
|
module Dev |
|
module Nix |
|
module Build |
|
# This class reads build output line-by-line from nix-build (or nix-shell, or whatever), |
|
# updating an internal state indicating progress, and emits a status object to a registered |
|
# callback (which must not mutate it) each time that status changes. |
|
# |
|
# Note that in order to provide better fidelity, we depend on running nix with `-vvvQ`, as this |
|
# is required to get messages when a derivation output *finished* downloading. |
|
# |
|
# Simple usage example: |
|
# |
|
# bop = Dev::Nix::BuildOutputParser.new { |status| puts status.to_s } |
|
# run_nix_and_yield_each_line('-vvvQ') { |line| bop.process(line) } |
|
# |
|
# The basic form of (this hyper-verbose) nix-build output that we parse is: |
|
# |
|
# (...snip...) |
|
# these derivations will be built: |
|
# /nix/store/${A} |
|
# these paths will be fetched (143.56 MiB download, 737.31 MiB unpacked): |
|
# /nix/store/${B} |
|
# /nix/store/${C} |
|
# (...snip...) |
|
# copying path '/nix/store/${B}' from 'https://cache.nixos.org'... |
|
# (...snip...) |
|
# building '/nix/store/${A}'... |
|
# (...snip...) |
|
# copying path '/nix/store/${C}' from 'https://cache.nixos.org'... |
|
# (...snip...) |
|
# substitution of path '/nix/store/${C}' succeeded |
|
# (...snip...) |
|
# substitution of path '/nix/store/${B}' succeeded |
|
# (...snip...) |
|
# builder process for '/nix/store/${A}' finished |
|
# (...snip...) |
|
# |
|
# With this input, our callback would fire at every "(...snip...)" except the first. |
|
# |
|
# -vv logs almost work, but there's no event that lets us detect when a build (as opposed to a |
|
# fetch) has completed. -Q is not strictly necessary but eliminates output generated by |
|
# derivations, more carefully guaranteeing that we won't get confounding inputs. |
|
class OutputParser |
|
extend(Dev::Util::PrivateConstants) |
|
|
|
Error = Class.new(StandardError) |
|
|
|
private_constants do |
|
STATE_INIT = :init |
|
STATE_LISTING_BUILT = :listing_built |
|
STATE_LISTING_FETCHED = :listing_fetched |
|
STATE_WORKING = :working |
|
STATE_AWAITING_BUILD_FAILURE = :awaiting_build_failure |
|
|
|
PAT_WILL_BE_BUILT = %r{^these derivations will be built} |
|
PAT_WILL_BE_FETCHED = %r{^these (derivations|paths) will be fetched \((.*) download, (.*) unpacked\)} |
|
PAT_INDENTED_STORE_PATH = %r{\s+(/nix/store/.*)} |
|
PAT_COPYING = %r{^copying path '(.*?)'} |
|
PAT_BUILDING = %r{^building '(.*?)'\.\.\.} |
|
PAT_FETCH_SUCCESS = %r{^substitution of path '(.*?)' succeeded} |
|
PAT_BUILD_DONE = %r{^builder process for '(.*?)' finished} |
|
PAT_BUILD_FAILED = %r{^builder for '(.*?)' failed} |
|
PAT_FETCH_FAILED = %r{^path '(.*?)' is required, but there is no substituter that can build it} |
|
|
|
# user-environment is a sort of special derivation name used by nix-build that doesn't show up |
|
# in the initial preamble listing of entries that will be built or fetched. It only really |
|
# shows up at higher levels of verbosity, so we're just going to ignore it completely. |
|
PAT_USER_ENVIRONMENT = %r{^/nix/store/[a-z0-9]{32}-user-environment(\.drv)?$} |
|
|
|
REPROCESS = true |
|
NO_REPROCESS = false |
|
|
|
EMIT = true |
|
NO_EMIT = false |
|
end |
|
|
|
def initialize(&block) |
|
@state = STATE_INIT |
|
@lineno = 0 |
|
@status = Dev::Nix::Build::Status.new |
|
@callback = block |
|
end |
|
|
|
def process(line) |
|
@lineno += 1 |
|
# On transition, we set @reprocess to trigger the line to be reevaluated in the new state. |
|
@reprocess = true |
|
while @reprocess |
|
@reprocess = false |
|
_process(line) |
|
end |
|
rescue Error, Build::Status::UnallowableTransition => e |
|
error = Dev::Bug.new("error on line #{@lineno}: #{e.message}") |
|
error.set_backtrace([caller[0]] + e.backtrace) |
|
raise(error) |
|
end |
|
|
|
private |
|
|
|
# rubocop:disable Layout/SpaceBeforeSemicolon,Style/WhenThen,Style/PerlBackrefs,Style/Semicolon |
|
# rubocop:disable Style/IdenticalConditionalBranches,Style/AndOr,Metrics/LineLength |
|
def _process(line) |
|
case @state |
|
when STATE_INIT |
|
# Read a whoooooole bunch of input, then an listing of what will be build and fetched. |
|
case line |
|
when PAT_WILL_BE_BUILT ; transition(STATE_LISTING_BUILT, NO_REPROCESS) |
|
when PAT_WILL_BE_FETCHED ; transition(STATE_LISTING_FETCHED, NO_REPROCESS) |
|
else ; noop |
|
end |
|
when STATE_LISTING_BUILT |
|
# Don't actually emit while we read these in: they all come immediately as a batch, so |
|
# instead, emit a record when we exit this state. |
|
case line |
|
when PAT_WILL_BE_FETCHED ; transition(STATE_LISTING_FETCHED, NO_REPROCESS) |
|
when PAT_INDENTED_STORE_PATH ; record_waiting_build($1) # (defer emit) |
|
else ; transition(STATE_WORKING, REPROCESS) ; emit |
|
end |
|
when STATE_LISTING_FETCHED |
|
# Don't actually emit while we read these in: they all come immediately as a batch, so |
|
# instead, emit a record when we exit this state. |
|
case line |
|
when PAT_WILL_BE_BUILT ; transition(STATE_LISTING_BUILT, NO_REPROCESS) |
|
when PAT_INDENTED_STORE_PATH ; record_waiting_fetch($1) # (defer emit) |
|
else ; transition(STATE_WORKING, REPROCESS) ; emit |
|
end |
|
when STATE_WORKING |
|
# We're reading a very verbose stream of build/fetch output. The odd message will |
|
# indicate the beginning or successful/unsuccessful termination of a build or fetch. |
|
case line |
|
when PAT_COPYING ; record_running_fetch($1) and emit |
|
when PAT_BUILDING ; record_running_build($1) and emit |
|
when PAT_FETCH_SUCCESS ; record_successful_fetch($1) and emit |
|
when PAT_FETCH_FAILED ; record_failed_fetch($1) and emit |
|
when PAT_BUILD_DONE ; await_build_failure($1) |
|
else ; noop |
|
end |
|
when STATE_AWAITING_BUILD_FAILURE |
|
# If the line immediately following a "completed" build indicates a failure, the build |
|
# failed. Otherwise, it succeeded. |
|
case line |
|
when PAT_BUILD_FAILED ; record_failed_build(@await) and emit ; transition(STATE_WORKING, REPROCESS) |
|
else ; record_successful_build(@await) and emit ; transition(STATE_WORKING, REPROCESS) |
|
end |
|
end |
|
end |
|
# rubocop:enable Layout/SpaceBeforeSemicolon,Style/WhenThen,Style/PerlBackrefs,Style/Semicolon |
|
# rubocop:enable Style/IdenticalConditionalBranches,Style/AndOr,Metrics/LineLength |
|
|
|
def emit |
|
@callback.call(@status) |
|
end |
|
|
|
def transition(next_state, reprocess) |
|
@state = next_state |
|
@reprocess = reprocess |
|
end |
|
|
|
def await_build_failure(store_path) |
|
@await = store_path |
|
transition(STATE_AWAITING_BUILD_FAILURE, NO_REPROCESS) |
|
end |
|
|
|
def noop |
|
end |
|
|
|
def record_waiting_build(store_path) |
|
return unless check_store_path(store_path) |
|
@status.move_build(store_path, to: Status::WAITING) |
|
EMIT |
|
end |
|
|
|
def record_waiting_fetch(store_path) |
|
return unless check_store_path(store_path) |
|
@status.move_fetch(store_path, to: Status::WAITING) |
|
EMIT |
|
end |
|
|
|
def record_running_fetch(store_path) |
|
return unless check_store_path(store_path) |
|
@status.move_fetch(store_path, to: Status::RUNNING) |
|
EMIT |
|
end |
|
|
|
def record_running_build(store_path) |
|
return unless check_store_path(store_path) |
|
@status.move_build(store_path, to: Status::RUNNING) |
|
EMIT |
|
end |
|
|
|
def record_successful_fetch(store_path) |
|
return unless check_store_path(store_path) |
|
|
|
# The line we use to recognize successful fetches occurs in failed |
|
# ones too, after the failure message. |
|
return(NO_EMIT) if @status.fetch_state(store_path) == Status::FAILED |
|
|
|
@status.move_fetch(store_path, to: Status::SUCCEEDED) |
|
EMIT |
|
end |
|
|
|
def record_successful_build(store_path) |
|
return unless check_store_path(store_path) |
|
@status.move_build(store_path, to: Status::SUCCEEDED) |
|
EMIT |
|
end |
|
|
|
def record_failed_fetch(store_path) |
|
return unless check_store_path(store_path) |
|
|
|
# Nix sometimes (optimistically?) tries to fetch things (apparently |
|
# relating to builds) that aren't listed in our list of fetches. |
|
state = @status.fetch_state(store_path) |
|
unless state == Status::WAITING || state == Status::RUNNING |
|
return(NO_EMIT) |
|
end |
|
|
|
@status.move_fetch(store_path, to: Status::FAILED) |
|
EMIT |
|
end |
|
|
|
def record_failed_build(store_path) |
|
return unless check_store_path(store_path) |
|
@status.move_build(store_path, to: Status::FAILED) |
|
EMIT |
|
end |
|
|
|
def check_store_path(store_path) |
|
raise(Error, 'failed to parse store path from line') if store_path.nil? |
|
return(false) if store_path =~ PAT_USER_ENVIRONMENT |
|
true |
|
end |
|
end |
|
end |
|
end |
|
end |