| # |
| # Bootstrap utility functions. |
| # |
| |
| package Bootstrap::Util; |
| |
| use File::Temp qw(tempfile tempdir); |
| use File::Spec::Functions; |
| use MozBuild::Util qw(RunShellCommand); |
| use base qw(Exporter); |
| |
| our @EXPORT_OK = qw(CvsCatfile CvsTag |
| GetPushRepo |
| GetDiffFileList |
| GetFtpNightlyDir |
| LoadLocaleManifest |
| GetEqualPlatforms |
| GetLocaleManifest |
| GetBouncerPlatforms GetPatcherPlatforms |
| GetBouncerToPatcherPlatformMap |
| GetBuildbotToFTPPlatformMap |
| GetFTPToBuildbotPlatformMap); |
| |
| our($DEFAULT_SHELL_TIMEOUT); |
| |
| use strict; |
| |
| # This maps Bouncer platforms, used in bouncer and the shipped-locales file |
| # to patcher2 platforms used in... patcher2. They're different for some |
| # historical reason, which should be considered a bug. |
| # |
| # This is somewhat incomplete, as Bouncer has the 'osxppc' platform and |
| # patcher2 has the 'unimac' platform, neither of which we need any more |
| # beacuse we don't need to disambiguate between PPC mac and universal binaries |
| # anymore. |
| # |
| # Also, bouncer uses "win", not win32; shipped-locales uses win32. They |
| # couldn't be the same, of course. |
| |
| my %PLATFORM_MAP = (# bouncer/shipped-locales platform => patcher2 platform |
| 'win32' => 'win32', |
| 'win64' => 'win64', |
| 'linux' => 'linux-i686', |
| 'linux64' => 'linux-x86_64', |
| 'osx' => 'mac', |
| 'osxppc' => 'macppc'); |
| |
| my %PLATFORM_FTP_MAP = (# buildbot platform => ftp directory |
| 'linux' => 'linux-i686', |
| 'linux64' => 'linux-x86_64', |
| 'macosx' => 'mac', |
| 'macosx64' => 'mac', |
| 'win32' => 'win32', |
| 'win64' => 'win64'); |
| |
| my %EQUAL_PLATFORMS = ('linux' => ['linux64'], |
| 'win32' => ['win64']); |
| |
| my $DEFAULT_CVSROOT = ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot'; |
| |
| $DEFAULT_SHELL_TIMEOUT = 3600; |
| |
| ## |
| # Turn an array of directory/filenames into a CVS module path. |
| # ( name comes from File::Spec::catfile() ) |
| # |
| # Note that this function does not take any arguments, to make the usage |
| # more like File::Spec::catfile() |
| ## |
| sub CvsCatfile { |
| return join('/', @_); |
| } |
| |
| sub GetBouncerToPatcherPlatformMap { |
| return %PLATFORM_MAP; |
| } |
| sub GetBuildbotToFTPPlatformMap { |
| return %PLATFORM_FTP_MAP; |
| } |
| |
| sub GetBouncerPlatforms { |
| return keys(%PLATFORM_MAP); |
| } |
| |
| sub GetEqualPlatforms { |
| my $platform = shift; |
| return $EQUAL_PLATFORMS{$platform} || 0; |
| } |
| |
| sub GetFTPToBuildbotPlatformMap { |
| my %ret; |
| while (my ($key, $value) = each %PLATFORM_FTP_MAP){ |
| # Special case for platforms with overlapping ftp names. macosx and |
| # macosx64 buildbot platforms become "mac" on FTP. To prevent |
| # unexpected reverse mapping for the following hash, we ignore macosx |
| # here. macosx is not supported by Firefox 4.x release automation, but |
| # still may be used for some internal conversions. |
| if ($key eq 'macosx'){ |
| next; |
| } |
| $ret{$value}=$key; |
| } |
| return %ret; |
| } |
| |
| ## |
| # GetFtpNightlyDir - construct the FTP path for pushing builds & updates to |
| # returns scalar |
| # |
| # no mandatory arguments |
| ## |
| |
| sub GetFtpNightlyDir { |
| my $config = new Bootstrap::Config(); |
| my $product = $config->Get(var => 'product'); |
| |
| my $nightlyDir = CvsCatfile('/home', 'ftp', 'pub', $product, 'nightly') . '/'; |
| return $nightlyDir; |
| } |
| |
| sub GetPatcherPlatforms { |
| return values(%PLATFORM_MAP); |
| } |
| |
| # Loads and parses the shipped-locales manifest file, so get hash of |
| # locale -> [bouncer] platform mappings; returns success/failure in |
| # reading/parsing the locales file. |
| |
| sub LoadLocaleManifest { |
| my %args = @_; |
| |
| die "ASSERT: LoadLocaleManifest(): needs a HASH ref" if |
| (not exists($args{'localeHashRef'}) or |
| ref($args{'localeHashRef'}) ne 'HASH'); |
| |
| my $localeHash = $args{'localeHashRef'}; |
| my $manifestFile = $args{'manifest'}; |
| |
| if (not -e $manifestFile) { |
| die ("ASSERT: Bootstrap::Util::LoadLocaleManifest(): Can't find manifest" |
| . " $manifestFile"); |
| } |
| |
| open(MANIFEST, "<$manifestFile") or return 0; |
| my @manifestLines = <MANIFEST>; |
| close(MANIFEST); |
| |
| my @bouncerPlatforms = GetBouncerPlatforms(); |
| @bouncerPlatforms = sort(@bouncerPlatforms); |
| |
| foreach my $line (@manifestLines) { |
| # We create an instance variable so if the caller munges the reference |
| # to which a certain locale points, they won't screw up all the other |
| # locales; previously, this was a shared array ref, which is bad. |
| my @bouncerPlatformsInstance = @bouncerPlatforms; |
| |
| my @elements = split(/\s+/, $line); |
| # Grab the locale; we do it this way, so we can use the rest of the |
| # array as a platform list, if we need it... |
| my $locale = shift(@elements); |
| @elements = sort(@elements); |
| |
| # We don't add a $ on the end, because of things like ja-JP-mac |
| if ($locale !~ /^[a-z]{2}(\-[A-Z]{2})?/) { |
| die "ASSERT: invalid locale in manifest file: $locale"; |
| } |
| |
| # So this is kinda weird; if the locales are followed by a platform, |
| # then they don't exist for all the platforms; if they're all by their |
| # lonesome, then they exist for all platforms. So, since we shifted off |
| # the locale above, if there's anything left in the array, it's the |
| # platforms that are valid for this locale; if there isn't, then that |
| # platform is valid for all locales. |
| $localeHash->{$locale} = scalar(@elements) ? \@elements : |
| \@bouncerPlatformsInstance; |
| |
| foreach my $platform (@{$localeHash->{$locale}}) { |
| die "ASSERT: invalid platform: $platform" if |
| (not grep($platform eq $_, @bouncerPlatforms)); |
| } |
| } |
| |
| # Add en-US, which isn't in shipped-locales, because it's assumed that |
| # we always need en-US, for all platforms, to ship. |
| $localeHash->{'en-US'} = \@bouncerPlatforms; |
| return 1; |
| } |
| |
| |
| sub GetLocaleManifest { |
| my %args = @_; |
| |
| my $mozillaCvsroot = $args{'cvsroot'} || $DEFAULT_CVSROOT; |
| # XXX - The cruel joke is that this default value won't work; there is |
| # no shipped-locales file on the trunk... yet... |
| my $releaseTag = $args{'tag'} || 'HEAD'; |
| my $appName = $args{'app'}; |
| |
| my $localeManifest = {}; |
| |
| # Remove unshipped files/locales and set proper mode on dirs; start |
| # by checking out the shipped-locales file |
| $ENV{'CVS_RSH'} = 'ssh'; |
| |
| my ($shippedLocalesTmpHandle, $shippedLocalesTmpFile) = tempfile(); |
| $shippedLocalesTmpHandle->close(); |
| |
| # We dump stderr here so we can ignore having to parse all of the |
| # RCS info that cvs dumps when you co with -p |
| my $rv = RunShellCommand(command => 'cvs', |
| args => ['-d', $mozillaCvsroot, |
| 'co', '-p', |
| '-r', $releaseTag, |
| CvsCatfile('mozilla', $appName, |
| 'locales', 'shipped-locales')], |
| redirectStderr => 0, |
| logfile => $shippedLocalesTmpFile); |
| |
| if ($rv->{'exitValue'} != 0) { |
| die "ASSERT: GetLocaleManifest(): shipped-locale checkout failed\n"; |
| } |
| |
| if (not LoadLocaleManifest(localeHashRef => $localeManifest, |
| manifest => $shippedLocalesTmpFile)) { |
| die "Bootstrap::Util: GetLocaleManifest() failed to load manifest\n"; |
| } |
| |
| return $localeManifest; |
| } |
| |
| sub CvsTag { |
| my %args = @_; |
| |
| # All the required args first, followed by the optional ones... |
| die "ASSERT: Bootstrap::Step::Tag::CvsTag(): null tagName" if |
| (!exists($args{'tagName'})); |
| my $tagName = $args{'tagName'}; |
| |
| die "ASSERT: Bootstrap::Step::Tag::CvsTag(): null cvsDir" if |
| (!exists($args{'cvsDir'})); |
| my $cvsDir = $args{'cvsDir'}; |
| |
| die "ASSERT: Bootstrap::Step::Tag::CvsTag(): invalid files data" if |
| (exists($args{'files'}) && ref($args{'files'}) ne 'ARRAY'); |
| |
| die "ASSERT: Bootstrap::Step::Tag::CvsTag(): null logFile" |
| if (!exists($args{'logFile'})); |
| my $logFile = $args{'logFile'}; |
| |
| my $branch = exists($args{'branch'}) ? $args{'branch'} : 0; |
| my $files = exists($args{'files'}) ? $args{'files'} : []; |
| my $force = exists($args{'force'}) ? $args{'force'} : 0; |
| my $timeout = exists($args{'timeout'}) ? $args{'timeout'} : |
| $DEFAULT_SHELL_TIMEOUT; |
| |
| # only force or branch specific files, not the whole tree |
| if ($force && scalar(@{$files}) <= 0) { |
| die("ASSERT: Bootstrap::Util::CvsTag(): Cannot specify force without files"); |
| } elsif ($branch && scalar(@{$files}) <= 0) { |
| die("ASSERT: Bootstrap::UtilCvsTag(): Cannot specify branch without files"); |
| } elsif ($branch && $force) { |
| die("ASSERT: Bootstrap::UtilCvsTag(): Cannot specify both branch and force"); |
| } |
| |
| my @cmdArgs; |
| push(@cmdArgs, 'tag'); |
| push(@cmdArgs, '-F') if ($force); |
| push(@cmdArgs, '-b') if ($branch); |
| push(@cmdArgs, $tagName); |
| push(@cmdArgs, @{$files}) if (scalar(@{$files}) > 0); |
| |
| # We can't use Bootstrap::Step logs since we are in Util, oh well... |
| print 'log: Running "cvs tag" as follows in' . $cvsDir . ":\n"; |
| print 'log: cvs ' . join(' ', @cmdArgs) . "\n"; |
| print 'log: Logging output to: ' . $logFile . "\n"; |
| print 'log: Timeout: ' . $timeout . "\n"; |
| |
| my %cvsTagArgs = (command => 'cvs', |
| args => \@cmdArgs, |
| dir => $cvsDir, |
| logfile => $logFile); |
| |
| $cvsTagArgs{'timeout'} = $timeout if (defined($timeout)); |
| $cvsTagArgs{'output'} = $args{'output'} if (exists($args{'output'})); |
| |
| return RunShellCommand(%cvsTagArgs); |
| } |
| |
| sub GetPushRepo { |
| my %args = @_; |
| my ($repo, $pushRepo); |
| |
| # Required arguments |
| die "ASSERT: Bootstrap::Util::GetPushRepo(): null repo" if |
| (!exists($args{'repo'})); |
| $pushRepo = $repo = $args{'repo'}; |
| |
| $pushRepo =~ s/^https?/ssh/; |
| if ($pushRepo !~ m/^ssh/) { |
| die "ASSERT: Bootstrap::Util::GetPushRepo(): could not generate " . |
| "push repo for: $repo"; |
| } |
| return $pushRepo; |
| } |
| |
| sub GetDiffFileList { |
| my %args = @_; |
| |
| foreach my $requiredArg (qw(cvsDir prevTag newTag)) { |
| if (!exists($args{$requiredArg})) { |
| die "ASSERT: MozBuild::Util::GetDiffFileList(): null arg: " . |
| $requiredArg; |
| } |
| } |
| |
| my $cvsDir = $args{'cvsDir'}; |
| my $firstTag = $args{'prevTag'}; |
| my $newTag = $args{'newTag'}; |
| |
| my $rv = RunShellCommand(command => 'cvs', |
| args => ['-q', 'diff', '-uN', |
| '-r', $firstTag, |
| '-r', $newTag], |
| dir => $cvsDir, |
| timeout => 3600); |
| |
| # Gah. So, the shell return value of "cvs diff" is dependent on whether or |
| # not there were diffs, NOT whether or not the command succeeded. (Thanks, |
| # CVS!) So, we can't really check exitValue here, since it could be 1 or |
| # 0, depending on whether or not there were diffs (and both cases are valid |
| # for this function). Maybe if there's an error it returns a -1? Or 2? |
| # Who knows. |
| # |
| # So basically, we check that it's not 1 or 0, which... isn't a great test. |
| # |
| # TODO - check to see if timedOut, dumpedCore, or sigNum are set. |
| if ($rv->{'exitValue'} != 1 && $rv->{'exitValue'} != 0) { |
| die("ASSERT: MozBuild::Util::GetDiffFileList(): cvs diff returned " . |
| $rv->{'exitValue'}); |
| } |
| |
| my @differentFiles = (); |
| |
| foreach my $line (split(/\n/, $rv->{'output'})) { |
| if ($line =~ /^Index:\s(.+)$/) { |
| push(@differentFiles, $1); |
| } |
| } |
| |
| |
| return \@differentFiles; |
| } |
| |
| 1; |