Nanako/DHDAXCW-Rockchip-OpenWrt
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

download.pl: add aria2c support

Browse Source 
Use aria2c download tool by default on package download if available in
the system.
aria2c permits to use multiple mirrors and may improve download speed on
special context where servers are hard to reach.

Co-authored-by: Christian Marangi <ansuelsmth@gmail.com>
Signed-off-by: Bradford Zhang <zyc@zyc.name>
[ fix wrong var in the script and improve commit description ]
Signed-off-by: Christian Marangi <ansuelsmth@gmail.com>
It is hereby declared that changes to aria2c download earlier from embedfire
This commit is contained in:
DHDAXCW 2022-09-28 02:30:34 +00:00
parent 79ac858356
commit ff9b74749e
4 changed files with 49 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -17,7 +17,7 @@
sudo apt install -y ack antlr3 asciidoc autoconf automake autopoint binutils bison build-essential \
bzip2 ccache cmake cpio curl device-tree-compiler fastjar flex gawk gettext gcc-multilib g++-multilib \
git gperf haveged help2man intltool libc6-dev-i386 libelf-dev libglib2.0-dev libgmp3-dev libltdl-dev \
libmpc-dev libmpfr-dev libncurses5-dev libncursesw5-dev libreadline-dev libssl-dev libtool lrzsz \
libmpc-dev libmpfr-dev libncurses5-dev libncursesw5-dev libreadline-dev libssl-dev libtool lrzsz aria2 \
mkisofs msmtp nano ninja-build p7zip p7zip-full patch pkgconf python2.7 python3 python3-pip libpython3-dev qemu-utils \
rsync scons squashfs-tools subversion swig texinfo uglifyjs upx-ucl unzip vim wget xmlto xxd zlib1g-dev libfuse-dev
```

4
config/Config-build.in
View File

@ -58,6 +58,10 @@ menu "Global build settings"
bool "Enable signature checking in opkg"
default SIGNED_PACKAGES
config DOWNLOAD_CHECK_CERTIFICATE
bool "Enable TLS certificate verification during package download"
default y
comment "General build options"
config TESTING_KERNEL

3
rules.mk
View File

@ -269,6 +269,9 @@ ESED:=$(STAGING_DIR_HOST)/bin/sed -E -i -e
MKHASH:=$(STAGING_DIR_HOST)/bin/mkhash
# MKHASH is used in /scripts, so we export it here.
export MKHASH
# DOWNLOAD_CHECK_CERTIFICATE is used in /scripts, so we export it here.
DOWNLOAD_CHECK_CERTIFICATE:=$(CONFIG_DOWNLOAD_CHECK_CERTIFICATE)
export DOWNLOAD_CHECK_CERTIFICATE
CP:=cp -fpR
LN:=ln -sf
XARGS:=xargs -r

50
scripts/download.pl
View File

@ -24,6 +24,8 @@ my $scriptdir = dirname($0);
my @mirrors;
my $ok;
my $check_certificate = $ENV{DOWNLOAD_CHECK_CERTIFICATE} eq "y";
$url_filename or $url_filename = $filename;
sub localmirrors {
@ -70,21 +72,50 @@ sub hash_cmd() {
return undef;
}
sub download_cmd($) {
sub download_cmd {
my $url = shift;
my $have_curl = 0;
my $have_aria2c = 0;
my $filename = shift;
my $additional_mirrors = join(" ", map "$_/$filename", @_);
my @chArray = ('a'..'z', 'A'..'Z', 0..9);
my $rfn = join '', "${filename}_", map{ $chArray[int rand @chArray] } 0..9;
if (open CURL, '-|', 'curl', '--version') {
if (defined(my $line = readline CURL)) {
$have_curl = 1 if $line =~ /^curl /;
}
close CURL;
}
if (open ARIA2C, '-|', 'aria2c', '--version') {
if (defined(my $line = readline ARIA2C)) {
$have_aria2c = 1 if $line =~ /^aria2 /;
}
close ARIA2C;
}
return $have_curl
? (qw(curl -f --connect-timeout 20 --retry 5 --location --insecure), shellwords($ENV{CURL_OPTIONS} || ''), $url)
: (qw(wget --tries=5 --timeout=20 --no-check-certificate --output-document=-), shellwords($ENV{WGET_OPTIONS} || ''), $url)
;
if ($have_aria2c) {
@mirrors=();
return join(" ", "[ -d $ENV{'TMPDIR'}/aria2c ] || mkdir $ENV{'TMPDIR'}/aria2c;",
"touch $ENV{'TMPDIR'}/aria2c/${rfn}_spp;",
qw(aria2c --stderr -c -x2 -s10 -j10 -k1M), $url, $additional_mirrors,
$check_certificate ? () : '--check-certificate=false',
"--server-stat-of=$ENV{'TMPDIR'}/aria2c/${rfn}_spp",
"--server-stat-if=$ENV{'TMPDIR'}/aria2c/${rfn}_spp",
"-d $ENV{'TMPDIR'}/aria2c -o $rfn;",
"cat $ENV{'TMPDIR'}/aria2c/$rfn;",
"rm $ENV{'TMPDIR'}/aria2c/$rfn $ENV{'TMPDIR'}/aria2c/${rfn}_spp");
} elsif ($have_curl) {
return (qw(curl -f --connect-timeout 20 --retry 5 --location),
$check_certificate ? () : '--insecure',
shellwords($ENV{CURL_OPTIONS} || ''),
$url);
} else {
return (qw(wget --tries=5 --timeout=20 --output-document=-),
$check_certificate ? () : '--no-check-certificate',
shellwords($ENV{WGET_OPTIONS} || ''),
$url);
}
}
my $hash_cmd = hash_cmd();
@ -94,6 +125,7 @@ sub download
{
my $mirror = shift;
my $download_filename = shift;
my @additional_mirrors = @_;
$mirror =~ s!/$!!;
@ -140,9 +172,9 @@ sub download
}
};
} else {
my @cmd = download_cmd("$mirror/$download_filename");
my @cmd = download_cmd("$mirror/$download_filename", $download_filename, @additional_mirrors);
print STDERR "+ ".join(" ",@cmd)."\n";
open(FETCH_FD, '-|', @cmd) or die "Cannot launch curl or wget.\n";
open(FETCH_FD, '-|', @cmd) or die "Cannot launch aria2c, curl or wget.\n";
$hash_cmd and do {
open MD5SUM, "| $hash_cmd > '$target/$filename.hash'" or die "Cannot launch $hash_cmd.\n";
};
@ -301,9 +333,9 @@ while (!-f "$target/$filename") {
my $mirror = shift @mirrors;
$mirror or die "No more mirrors to try - giving up.\n";
download($mirror, $url_filename);
download($mirror, $url_filename, @mirrors);
if (!-f "$target/$filename" && $url_filename ne $filename) {
download($mirror, $filename);
download($mirror, $filename, @mirrors);
}
}