#!/usr/bin/perl use strict; use warnings; use autodie qw(:all); use feature qw(:5.14); use LWP::UserAgent; use LWP::ConnCache; use JSON; use URI; use Text::Wrap qw(wrap $columns); use File::Basename qw(dirname); our $packager = ''; my %pkgmap = (); my %licenses = (); my $template = <<'EOF'; # Automatically generated by apkbuild-pypi, template 4 [% authors %] pkgname=[% pkgname %] pkgver=[% pkgver %] pkgrel=[% pkgrel %] #_pkgreal is used by apkbuild-pypi to find modules at PyPI _pkgreal=[% pkgreal %] pkgdesc="[% pkgdesc %]" url="[% url %]" arch="noarch" license="[% license %]" depends="" makedepends="py3-gpep517 py3-setuptools py3-wheel" checkdepends="py3-pytest" subpackages="$pkgname-pyc" source="[% source %]" builddir="[% builddir %]" options="[% options %]"[% options_comment %] [% compatibility %] build() { gpep517 build-wheel \ --wheel-dir .dist \ --output-fd 3 3>&1 >&2 } check() { python3 -m venv --clear --without-pip --system-site-packages .testenv .testenv/bin/python3 -m installer .dist/*.whl .testenv/bin/python3 -m pytest } package() { python3 -m installer -d "$pkgdir" \ .dist/*.whl } EOF my $ua = LWP::UserAgent->new(); my $json = JSON->new; $ua->env_proxy; $ua->conn_cache(LWP::ConnCache->new); sub read_file { my ($filename) = @_; open my $fh, '<:utf8', $filename; local $/; my $text = <$fh>; return $text; } sub read_assignments_from_file { my ($filename) = @_; return () if ( ! -e $filename ); my $text = read_file($filename); my %sline = $text =~ /^(\w+)\s*=\s*([^\"\n]*)$/mg; my %mline = $text =~ /^(\w+)\s*=\s*\"([^\"]*)\".*$/mg; my %hash = ( %sline, %mline ); return \%hash if $filename ne 'APKBUILD'; my $authors = join( "\n", $text =~ /^# Contributor: .*$/mg, $text =~ /^# Maintainer: .*$/mg ); $hash{'authors'} = $authors if length($authors) > 1; if ($text =~ m/^provides=\"(.*)\"(.*)$/m) { $hash{'provides'} = $1; $hash{'provides_comment'} = $2; } if ($text =~ m/^replaces=\"(.*)\"(.*)$/m) { $hash{'replaces'} = $1; $hash{'replaces_comment'} = $2; } # workaround for `builddir="$srcdir"/$_pkgname-$pkgver` if ($text =~ m/^builddir=\"(.*)\"([^\s]*)/m) { $hash{'builddir'} = $1 . $2; } if ($text =~ m/^options=\"(.*)\"(.*)$/m) { $hash{'options'} = $1; $hash{'options_comment'} = $2; } return \%hash; } sub map_pypi_to_apk { my ($pypi) = @_; return $pkgmap{$pypi} unless !exists($pkgmap{$pypi}); return 'py3-'.lc($pypi); } sub map_license { my ($license) = @_; $license //= ''; $license =~ s/ or / /g; return $license; } sub get_source { my ($distdata) = @_; my $pkgname = $distdata->{info}{name}; my $source; for my $url (@{$distdata->{urls}}) { if ($url->{python_version} eq 'source') { $source = URI->new($url->{url}); last; } } die "Unable to locate sources for $pkgname.\n" unless $source; my $filename = ($source->path_segments)[-1]; my $pretty_path = substr($pkgname, 0, 1) . "/$pkgname"; my $pretty_url = $source->clone; $pretty_url->path("/packages/source/$pretty_path/$filename"); my $response = $ua->head($pretty_url); if ($response->is_success) { return $pretty_url->as_string; } else { return $source->as_string; } } sub read_apkbuild { return read_assignments_from_file('APKBUILD'); } sub format_line { my $line = shift; return "\t" . $line . "\n"; } sub format_source { my $srcurl = shift; my $orig_src = shift; $orig_src =~ s/^\s+//mg; $orig_src =~ s/\s+/ /g; my @sources = split (/\s/, $orig_src); return $srcurl if @sources <= 1; shift @sources if $sources[0] =~ m/pkgver/; my $patches; for my $patch (@sources) { next if $patch eq ""; $patches .= format_line($patch); } return $srcurl . "\n" . ($patches // '') . "\t"; } sub write_apkbuild { my ($distdata, $apkbuild) = @_; my $replaces = undef; my $provides = undef; my $authors = undef; my $license = undef; my $url = undef; my $pkgname = undef; my $pkgdesc = undef; my $pkgrel = 0; my $builddir = undef; my $options = undef; my $options_comment = undef; my $orig_source = ""; if (our $use_homepage) { $url = $distdata->{info}{project_urls}{Homepage} || $distdata->{info}{home_page}; } if (defined $apkbuild) { $authors = $apkbuild->{authors}; $provides = $apkbuild->{provides}; $replaces = $apkbuild->{replaces}; $license = $apkbuild->{license}; $url = $apkbuild->{url}; $pkgname = $apkbuild->{pkgname}; $pkgdesc = $apkbuild->{pkgdesc}; $pkgrel = $apkbuild->{pkgrel}; $builddir = $apkbuild->{builddir}; $options = $apkbuild->{options} if defined $apkbuild->{options}; $options_comment = $apkbuild->{options_comment} if defined $apkbuild->{options_comment}; $orig_source = $apkbuild->{source}; if ($apkbuild->{pkgver} eq $distdata->{info}{version}) { $pkgrel++; } } my $pkgreal = $distdata->{info}{name}; my $srcurl = get_source($distdata); my %repl = ( authors => ($authors or "# Contributor: $packager\n# Maintainer: $packager"), pkgname => ($pkgname or map_pypi_to_apk($pkgreal)), pkgreal => $pkgreal, pkgver => $distdata->{info}{version}, pkgrel => $pkgrel, source => format_source($srcurl, $orig_source), license => ($license or map_license($distdata->{info}{license})), url => ($url or "https://pypi.org/project/${pkgreal}/"), pkgdesc => ($pkgdesc or $distdata->{info}{summary} or "Python module for $pkgreal"), builddir => ($builddir or ''), options => ($options or ''), options_comment => ($options_comment or ''), ); $repl{compatibility} = ""; if ($replaces) { my $comment = $apkbuild->{'replaces_comment'} // ''; $repl{compatibility} .= "\nreplaces=\"$replaces\"" . $comment; } if ($provides) { my $comment = $apkbuild->{'provides_comment'} // ''; $repl{compatibility} .= "\nprovides=\"$provides\"" . $comment; } $repl{compatibility} .= "\n" if $replaces or $provides; $repl{source} =~ s/-$repl{pkgver}/-\$pkgver/g; $template =~ s/\[% (.*?) %\]/$repl{$1}/g; open my $fh, '>:utf8', 'APKBUILD'; print {$fh} $template; close $fh; say "Wrote $repl{pkgname}/APKBUILD"; return \%repl; } sub unpack_source { system('abuild checksum unpack'); } sub prepare_tree { my %options = @_; unpack_source if $options{unpack}; system('abuild prepare'); } sub find_package_name { my ($apkbuild) = @_; my $pkgreal = ''; if (exists $apkbuild->{_realname}) { $pkgreal = $apkbuild->{_realname}; } elsif (exists $apkbuild->{_pkgreal}) { $pkgreal = $apkbuild->{_pkgreal}; } elsif (exists $apkbuild->{_pkgname}) { $pkgreal = $apkbuild->{_pkgname}; } elsif (exists $apkbuild->{_name}) { $pkgreal = $apkbuild->{_name}; } elsif (exists $apkbuild->{_realpkgname}) { $pkgreal = $apkbuild->{_realpkgname}; } elsif (exists $apkbuild->{_pkg_real}) { $pkgreal = $apkbuild->{_pkg_real}; } elsif (exists $apkbuild->{source}) { $pkgreal = $apkbuild->{source}; $pkgreal =~ m/(\w+)-/; $pkgreal = $1; } else { print "No pkg real found\n"; die; } return $pkgreal; } sub get_data { my ($package) = @_; my $response = $ua->get("https://pypi.org/pypi/$package/json"); $response->is_success or die $response->status_line; my $distdata = $json->decode($response->decoded_content); return $distdata; } sub parse_requires_dist { my $reqs = shift; # Valid PyPI regex: https://peps.python.org/pep-0508/#names my $pypi_regex = qr/(?i)([A-Z0-9][A-Z0-9][A-Z0-9._-]*[A-Z0-9])/; my @depends = map { m/$pypi_regex/; $1 || () } grep { $_ !~ m/; extra ==/ } @$reqs; my @checkdeps = map { m/$pypi_regex/; $1 || () } grep { m/; extra == ["'](tests|pytest)["']/ } @$reqs; my %reqs = ( depends => \@depends, checkdeps => \@checkdeps, ); return \%reqs; } sub format_depends { my $deps = shift; $columns = 102; $deps =~ s/ {2,}/ /g; $deps =~ s/^\s//g; $deps =~ s/\s$//g; if ( length($deps) >= $columns ) { $deps = wrap( "\t", "\t", $deps ); } $deps =~ s/\s$//g; if ( length($deps) >= $columns ) { $deps = "\n" . $deps . "\n\t"; } return $deps; } sub get_deps { my ($distdata, $data) = @_; my $reqs = parse_requires_dist($distdata->{info}{requires_dist}); my %depends = ('py3-pytest' => 'py3-pytest'); my @depends = map { my $apkname = map_pypi_to_apk($_); if (exists $depends{$apkname}) { () } else { $depends{$apkname} = $apkname } } @{$reqs->{depends}}; my @checkdeps = map { my $apkname = map_pypi_to_apk($_); exists($depends{$apkname}) ? () : $apkname } @{$reqs->{checkdeps}}; my $apk = read_file('APKBUILD'); my $depends = format_depends(join ' ', @depends); $apk =~ s/^depends=""/depends="$depends"/m; unshift @checkdeps, 'py3-pytest'; my $checkdeps = format_depends(join ' ', @checkdeps); $apk =~ s/^checkdepends="py3-pytest"/checkdepends="$checkdeps"/m; # remove empty variables $apk =~ s/.*="".{0,}\n//g; open my $fh, '>:utf8', 'APKBUILD'; print $fh $apk; say "Requires: @depends\n\nCheckDepends: @checkdeps"; } sub write_old_deps { my ($data, $apkbuild) = @_; my $apk = read_file('APKBUILD'); if (my $depends = $apkbuild->{depends}) { $apk =~ s/^depends=".*"/depends="$depends"/m; } if (my $makedeps = $apkbuild->{makedepends}) { if ($makedeps =~ m/py3-gpep517/) { $apk =~ s/^makedepends=".*"/makedepends="$makedeps"/m; } else { $apk =~ s/^makedepends="(.*)"/makedepends="$1 $makedeps"/m; } } if (my $checkdeps = $apkbuild->{checkdepends}) { if ($checkdeps =~ m/py3-pytest/) { $apk =~ s/^checkdepends=".*"/checkdepends="$checkdeps"/m; } else { $apk =~ s/^checkdepends="(.*)"/checkdepends="$1 $checkdeps"/m; } } # remove empty variables $apk =~ s/.*="".{0,}\n//g; open my $fh, '>:utf8', 'APKBUILD'; print $fh $apk; } sub update_builddir { my $apkbuild = read_apkbuild; my $pkgreal = $apkbuild->{'_pkgreal'}; my $pkgname = $apkbuild->{pkgname}; my $pkgver = $apkbuild->{pkgver}; my $oldbuilddir = $apkbuild->{builddir}; my $build_path = glob(" src/*${pkgver}/pyproject.toml src/*${pkgver}/setup.py "); if ($build_path) { my $newbuilddir = dirname($build_path); $newbuilddir =~ s/src/\$srcdir/; $newbuilddir =~ s/$pkgreal/\$_pkgreal/; $newbuilddir =~ s/$pkgver/\$pkgver/; my $apk = read_file('APKBUILD'); if ($pkgname eq $pkgreal and $newbuilddir eq '$srcdir/$_pkgreal-$pkgver') { # this will be deleted by the remove empty # variables regex in get_deps/write_old_deps $apk =~ s/^builddir=".*"/builddir=""/m; $newbuilddir = ''; } elsif ($newbuilddir eq $oldbuilddir) { return; } else { $apk =~ s/^builddir=".*"/builddir="$newbuilddir"/m; } print "\n\$builddir redefined:\n\t", "OLD: $oldbuilddir, NEW: $newbuilddir\n\n"; open my $fh, '>:utf8', 'APKBUILD'; print $fh $apk; } } my $abuild_conf = read_assignments_from_file('/etc/abuild.conf'); $packager = $abuild_conf->{PACKAGER} if $abuild_conf->{PACKAGER}; my $user_abuild_conf = read_assignments_from_file($ENV{"HOME"} . "/.abuild/abuild.conf"); $packager = $user_abuild_conf->{PACKAGER} if $user_abuild_conf->{PACKAGER}; sub usage { say <<'EOF'; Usage: apkbuild-pypi [create [homepage] | check | recreate [deps] | upgrade | update] In the repository root: create : Creates an APKBUILD for create homepage: Creates an APKBUILD for with url= field set to project homepage, if available In the package root: check : Reports current & latest version of the package recreate [deps] : Recreates the APKBUILD [also recalculate dependencies] upgrade : Upgrades to the latest version of the package update : Updates APKBUILD metadata EOF } if (! defined $ARGV[0]) { die usage; } elsif ($ARGV[0] eq 'create') { my $package = $ARGV[1]; $package or die usage; my $distdata = get_data($package); my $apkname = map_pypi_to_apk($package); mkdir $apkname; chdir $apkname; if ($ARGV[2] and $ARGV[2] eq 'homepage') { our $use_homepage = 1; } my $data = write_apkbuild($distdata, undef); unpack_source; update_builddir; get_deps($distdata, $data); prepare_tree( unpack => 0 ); } elsif ($ARGV[0] eq 'recreate') { my $apkbuild = read_apkbuild; if (! defined $apkbuild->{_pkgreal}) { $apkbuild->{_pkgreal} = find_package_name($apkbuild); } my $distdata = get_data($apkbuild->{_pkgreal}); my $pkgver = $distdata->{info}{version} =~ s/^[^0-9]+//r; if ($pkgver ne $apkbuild->{pkgver}) { #Reset pkgrel on upgrade on recreate say "Upgrading PyPI module from $apkbuild->{pkgver} to $pkgver"; $apkbuild->{pkgrel}=0; } my $data = write_apkbuild($distdata, $apkbuild); unpack_source; update_builddir; if ($ARGV[1] and $ARGV[1] eq 'deps') { get_deps($distdata, $data); } else { write_old_deps($data, $apkbuild); } prepare_tree( unpack => 0 ); } elsif ($ARGV[0] eq 'upgrade') { my $apkbuild = read_apkbuild; if (! defined $apkbuild->{_pkgreal}) { $apkbuild->{_pkgreal} = find_package_name($apkbuild); } my $distdata = get_data($apkbuild->{_pkgreal}); my $pkgver = $distdata->{info}{version}; if ($pkgver ne $apkbuild->{pkgver}) { say "Upgrading PyPI package from $apkbuild->{pkgver} to $pkgver"; my $text = read_file('APKBUILD'); $text =~ s/^(pkgver)=.*$/$1=$pkgver/mg or die "Can't find pkgver line in APKBUILD"; $text =~ s/^(pkgrel)=.*$/$1=0/mg or die "Can't find pkgrel line in APKBUILD"; open my $fh, '>:utf8', 'APKBUILD'; print $fh $text; close $fh; } else { say "Already up to date with PyPI"; } } elsif ($ARGV[0] eq 'check') { my $apkbuild = read_apkbuild; if (! defined $apkbuild->{_pkgreal}) { $apkbuild->{_pkgreal} = find_package_name($apkbuild); } my $distdata = get_data($apkbuild->{_pkgreal}); my $pkgver = $distdata->{info}{version}; say "$apkbuild->{pkgname}: Latest version: $pkgver Packaged version: $apkbuild->{pkgver}"; if ($pkgver ne $apkbuild->{pkgver}) { exit(1); } } elsif ($ARGV[0] eq 'update') { prepare_tree( unpack => 1 ); } else { die usage; }