pbuild

#!/usr/bin/perl
################################################################
#
# Copyright (c) 2021 SUSE LLC
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 or 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program (see the file COPYING); if not, write to the
# Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#
################################################################

BEGIN {
  if (!$::ENV{'BUILD_DIR'} && $0 ne '-' && $0 ne '-e' && -e $0 && ! -e '/etc/build.conf') {
    use Cwd ();
    my $p = Cwd::abs_path($0);
    $::ENV{'BUILD_DIR'} = $p if $p =~ s/\/[^\/]+$// && $p ne '/usr/lib/build' && -d "$p/PBuild";
  }
  unshift @INC, ($::ENV{'BUILD_DIR'} && ! -e '/etc/build.conf' ? $::ENV{'BUILD_DIR'} : '/usr/lib/build');
}

use Data::Dumper;
use POSIX;
use Cwd ();

use Build;

use PBuild::Source;
use PBuild::Recipe;
use PBuild::RepoMgr;
use PBuild::LocalRepo;
use PBuild::Multibuild;
use PBuild::Link;
use PBuild::Checker;
use PBuild::Options;
use PBuild::BuildConfig;
use PBuild::Result;
use PBuild::Download;
use PBuild::Preset;

# parse options
my ($opts, @dirs) = PBuild::Options::parse_options(@ARGV);
PBuild::Options::usage(0) if $opts->{'help'};
die("Usage: pbuild [options] [dir]\n") if @dirs > 1;
my $dir = @dirs ? $dirs[0] : '.';
$dir = Cwd::abs_path($dir) if $dir !~ /^\//;

# set defaults
my $libbuild = $INC[0];
$opts->{'libbuild'} = $libbuild;
$opts->{'root'} ||= '/var/tmp/build-root';
$opts->{'root'} = Cwd::abs_path($opts->{'root'}) if $opts->{'root'} !~ /^\//;
$opts->{'configdir'} ||= "$libbuild/configs";
if (!$opts->{'hostarch'}) {
  $opts->{'hostarch'} = (POSIX::uname())[4];
  die("cannot determine hostarch\n") unless $opts->{'hostarch'};
  $opts->{'hostarch'} = 'armv6hl' if $opts->{'hostarch'} eq 'armv6l';
  $opts->{'hostarch'} = 'armv7hl' if $opts->{'hostarch'} eq 'armv7l';
}
$opts->{'arch'} ||= $opts->{'hostarch'};
$opts->{'buildjobs'} = 1 unless $opts->{'buildjobs'};
$opts->{'buildjobs'} = 32 if $opts->{'buildjobs'} > 32;


# read presets
my $preset = PBuild::Preset::read_presets($dir, $opts->{'preset'});
PBuild::Preset::apply_preset($opts, $preset) if $preset;

my $reponame = $opts->{'reponame'};
if (!$reponame && $opts->{'dist'}) {
  $reponame = $opts->{'dist'}->[-1];
  $reponame =~ s/.*\///;
  $reponame =~ s/\.conf$//;
}

my $myarch = $opts->{'arch'};

my $builddir = $reponame ? "$dir/_build.$reponame.$myarch" : "$dir/_build.$myarch";

if ($opts->{'result-code'} || $opts->{'result-pkg'}) {
  PBuild::Result::print_result($opts, $builddir);
  exit;
}

my @baseconfigs;
my $distcnt = 0;
my @baseobsrepos;
for my $dist (@{$opts->{'dist'} || []}) {
  $distcnt++;
  if ($dist =~ /^https?:\/\//) {
    my ($config) = PBuild::Download::fetch($dist);
    push @baseconfigs, $config;
  } elsif ($dist =~ /^obs:\//) {
    my $islast = $distcnt == @{$opts->{'dist'} || []} ? 1 : 0;
    my ($obsconfigs, $obsrepos) = PBuild::OBS::fetch_all_configs($dist, $opts, $islast);
    push @baseconfigs, reverse @$obsconfigs;
    push @baseobsrepos, @$obsrepos;
  } elsif ($dist =~ /^empty:/) {
    next;
  } elsif (-e "$dir/_configs/$opts->{'dist'}.conf") {
    push @baseconfigs, PBuild::Util::readstr("$dir/_configs/$dist.conf");
  } else {
    my $baseconfigfile = Build::find_config_file($dist, $opts->{'configdir'});
    push @baseconfigs, PBuild::Util::readstr($baseconfigfile);
  }
}
# set repos from obs, does not work well with "mixed" configs
push @{$opts->{'repo'}}, @baseobsrepos if @baseobsrepos && !$opts->{'repo'};

my $localconfig = -s "$dir/_config" ? PBuild::Util::readstr("$dir/_config") : '';
$localconfig = "\n%define _repository $reponame\n\n$localconfig" if $localconfig ne '';

my $buildconfig = PBuild::BuildConfig::combineconfigs(@baseconfigs, $localconfig);
if (!$buildconfig) {
  print("Please specify a distribution!\n\n");
  PBuild::Options::usage(1);
}

my $bconf = Build::read_config($myarch, [ split("\n", $buildconfig) ]);

# default to repo/registry from config if not set
push @{$opts->{'repo'}}, 'config:' unless @{$opts->{'repo'} || []};
push @{$opts->{'registry'}}, 'config:' unless @{$opts->{'registry'} || []};

# substitute config: with values from config
for (splice(@{$opts->{'repo'}})) {
  push @{$opts->{'repo'}}, $_;
  splice(@{$opts->{'repo'}}, -1, 1, @{$bconf->{'repourl'}}) if $_ eq 'config:';
}
for (splice(@{$opts->{'registry'}})) {
  push @{$opts->{'registry'}}, $_;
  splice(@{$opts->{'registry'}}, -1, 1, @{$bconf->{'registryurl'}}) if $_ eq 'config:';
}

# expand the zypp:// repo
PBuild::RemoteRepo::expand_zypp_repo($opts) if $opts->{'repo'};

print "searching for packages\n";
my @pkgs = PBuild::Source::find_packages($dir);
die("no packages found in '$dir'\n") unless @pkgs;
my $npkgs = @pkgs;
print "found $npkgs packages\n";

print "getting package information\n";
my %pkgsrc;
for my $pkg (@pkgs) {
  my $files = PBuild::Source::list_package("$dir/$pkg");
  $pkgsrc{$pkg} = {
    'pkg' => $pkg,
    'dir' => "$dir/$pkg",
    'files' => $files,
    'srcmd5' => PBuild::Source::calc_srcmd5($files),
  };
}

# handle local links and multibuild packages
print "expanding package links\n";
PBuild::Link::expand_links(\%pkgsrc);

print "expanding multibuild packages\n";
PBuild::Multibuild::expand_multibuilds(\%pkgsrc);

@pkgs = sort keys %pkgsrc;

# handle onlybuild/excludebuild from the build config
if (exists $bconf->{'buildflags:excludebuild'}) {
  my %excludebuild;
  /^excludebuild:(.*)$/s && ($excludebuild{$1} = 1) for @{$bconf->{'buildflags'} || []};
  if (%excludebuild) {
    for my $pkg (@pkgs) {
      my $p = $pkgsrc{$pkg};
      my $releasename = $p->{'releasename'} || $pkg;
      $p->{'error'} = "excluded:project config excludebuild list" if $excludebuild{$pkg} || $excludebuild{$releasename};
    }
  }
}
if (exists $bconf->{'buildflags:onlybuild'}) {
  my %onlybuild;
  /^onlybuild:(.*)$/s && ($onlybuild{$1} = 1) for @{$bconf->{'buildflags'} || []};
  if (%onlybuild) {
    for my $pkg (@pkgs) {
      my $p = $pkgsrc{$pkg};
      my $releasename = $p->{'releasename'} || $pkg;
      $p->{'error'} = "excluded:project config onlybuild list" unless $onlybuild{$pkg} || $onlybuild{$releasename};
    }
  }
}

# parse all recipes in the packages to get dependency information
print "parsing recipe files\n";
my %containertags;
my $buildtype = $bconf->{'type'} || '';
$buildtype = 'spec' if !$buildtype || $buildtype eq 'UNDEFINED';
for my $pkg (@pkgs) {
  my $p = $pkgsrc{$pkg};
  PBuild::Recipe::parse($bconf, $p, $buildtype, $myarch);
  if ($p->{'buildtype'} && ($p->{'buildtype'} eq 'kiwi' || $p->{'buildtype'} eq 'docker') && !$p->{'error'}) {
    my @containerdeps = grep {/^container:/} @{$p->{'dep'} || []};
    next unless @containerdeps;
    $containertags{substr($_, 10)} = 1 for @containerdeps;
  }
}

#FIXME
for my $pkg (@pkgs) {
  my $p = $pkgsrc{$pkg};
  $p->{'useforbuildenabled'} = 1;
}

# delete obsolete entries from builddir
PBuild::LocalRepo::cleanup_builddir($builddir, \%pkgsrc);

# setup the repositories and registries
my $repomgr = PBuild::RepoMgr::create();
my @repos;
print "fetching local repo metadata\n";
push @repos, $repomgr->addlocalrepo($bconf, $myarch, $builddir, \%pkgsrc);

print "fetching remote repo metadata\n";
for my $repourl (@{$opts->{'repo'}}) {
  push @repos, $repomgr->addremoterepo($bconf, $myarch, $builddir, $repourl, $buildtype, $opts);
}

if (@{$opts->{'registry'} || []} && %containertags) {
  print "fetching remote registry metadata\n";
  my @containertags = sort keys %containertags;
  for my $registry (@{$opts->{'registry'} || []}) {
    push @repos, $repomgr->addremoteregistry($bconf, $myarch, $builddir, $registry, \@containertags);
  }
}

# load lastcheck cache
my %lastcheck;
if (-s "$builddir/.pbuild/_lastcheck") {
  my $oldlastcheck = PBuild::Util::retrieve("$builddir/.pbuild/_lastcheck", 1) || {};
  for my $pkg (@pkgs) {
    my $old = $oldlastcheck->{$pkg};
    $lastcheck{$pkg} = $old if $old && length($old) > 96;
  }
}

# setup builders
my @builders;
for my $no (1..$opts->{'buildjobs'}) {
  my $broot = $opts->{'root'};
  if ($opts->{'buildjobs'} > 1) {
    $broot .= '/%I' if $broot !~ /%I/;
    $broot =~ s/%I/$no/g;
  }
  push @builders, {
    'name' => $no,
    'root' => $broot,
    'idx' => scalar(@builders),
    'nbuilders' => $opts->{'buildjobs'},
  };
}

my $ctx;
my $runs = 0;
# the big loop: while there is something to do
while (1) {
  # create and setup checker
  if (!$ctx) {
    $ctx = PBuild::Checker::create($bconf, $myarch, $buildtype, \%pkgsrc, $builddir, $opts, $repomgr);
    print "preparing package pool\n" unless $runs;
    $ctx->prepare(\@repos);
    print "expanding dependencies\n" unless $runs;
    $ctx->pkgexpand(@pkgs);
    print "sorting packages\n" unless $runs;
    @pkgs = $ctx->pkgsort(@pkgs);
  }
  $runs++;
  $ctx->{'buildconfig'} = $buildconfig;
  $ctx->{'lastcheck'} = \%lastcheck;

  # check status of all packages
  my $result = $ctx->pkgcheck(\@builders, @pkgs);

  # update on-disk data
  PBuild::Util::mkdir_p("$builddir/.pbuild");
  PBuild::Util::store("$builddir/.pbuild/._result.$$", "$builddir/.pbuild/_result", $result);
  PBuild::Util::store("$builddir/.pbuild/._lastcheck.$$", "$builddir/.pbuild/_lastcheck", \%lastcheck);

  # get list of building jobs
  my @building = map {$_->{'job'}} grep {$_->{'job'}} @builders;
  last unless @building;

  # wait for one job to finish
  my $job = PBuild::Job::waitjob(@building);
  for (@builders) {
    delete $_->{'job'} if $_->{'job'} && $_->{'job'} == $job;
  }
  # process finished job
  my ($code, $buildresult) = PBuild::Job::finishjob($job);
  my $p = $job->{'pdata'};
  my $duration = $job->{'endtime'} - $job->{'starttime'};
  $duration = sprintf("%d:%02d", int($duration / 60), $duration % 60);
  my $bid = ($job->{'nbuilders'} || 1) > 1 ? "$job->{'name'}: " : '';
  print "${bid}finished $p->{'pkg'}/$p->{'recipe'} after ${duration}s: $code\n";

  my $jobhist = PBuild::BuildResult::makejobhist($p, $code, $job->{'readytime'}, $job->{'starttime'}, $job->{'endtime'}, $job->{'reason'}, $job->{'hostarch'});
  PBuild::BuildResult::addjobhist($builddir, $jobhist);

  # integrate build artifacts and extra files
  my $bininfo = PBuild::BuildResult::integrate_job($builddir, $job, $code, $buildresult);

  # if the build was successful, update artifact information and the local repo
  if ($bininfo) {
    PBuild::LocalRepo::update_gbininfo($builddir, $p->{'pkg'}, $bininfo);
    if ($p->{'useforbuildenabled'}) {
      # update with new local bin information
      $repomgr->updatelocalrepo($bconf, $myarch, $builddir, \%pkgsrc);
      # we also need a new checker
      undef $ctx;
    }
  }
}

# say goodbye
print "\npbuild is done:\n";
exit PBuild::Result::print_result($opts, $builddir);