From ff9b74749e641873e095723091562794aee71e4c Mon Sep 17 00:00:00 2001 From: DHDAXCW Date: Wed, 28 Sep 2022 02:30:34 +0000 Subject: [PATCH] download.pl: add aria2c support Use aria2c download tool by default on package download if available in the system. aria2c permits to use multiple mirrors and may improve download speed on special context where servers are hard to reach. Co-authored-by: Christian Marangi Signed-off-by: Bradford Zhang [ fix wrong var in the script and improve commit description ] Signed-off-by: Christian Marangi It is hereby declared that changes to aria2c download earlier from embedfire --- README.md | 2 +- config/Config-build.in | 4 ++++ rules.mk | 3 +++ scripts/download.pl | 50 ++++++++++++++++++++++++++++++++++-------- 4 files changed, 49 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 6c8828a18..e7caaee7f 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ sudo apt install -y ack antlr3 asciidoc autoconf automake autopoint binutils bison build-essential \ bzip2 ccache cmake cpio curl device-tree-compiler fastjar flex gawk gettext gcc-multilib g++-multilib \ git gperf haveged help2man intltool libc6-dev-i386 libelf-dev libglib2.0-dev libgmp3-dev libltdl-dev \ - libmpc-dev libmpfr-dev libncurses5-dev libncursesw5-dev libreadline-dev libssl-dev libtool lrzsz \ + libmpc-dev libmpfr-dev libncurses5-dev libncursesw5-dev libreadline-dev libssl-dev libtool lrzsz aria2 \ mkisofs msmtp nano ninja-build p7zip p7zip-full patch pkgconf python2.7 python3 python3-pip libpython3-dev qemu-utils \ rsync scons squashfs-tools subversion swig texinfo uglifyjs upx-ucl unzip vim wget xmlto xxd zlib1g-dev libfuse-dev ``` diff --git a/config/Config-build.in b/config/Config-build.in index 342859b7c..196d4e67a 100644 --- a/config/Config-build.in +++ b/config/Config-build.in @@ -58,6 +58,10 @@ menu "Global build settings" bool "Enable signature checking in opkg" default SIGNED_PACKAGES + config DOWNLOAD_CHECK_CERTIFICATE + bool "Enable TLS certificate verification during package download" + default y + comment "General build options" config TESTING_KERNEL diff --git a/rules.mk b/rules.mk index dbb2396e6..0463b02c2 100644 --- a/rules.mk +++ b/rules.mk @@ -269,6 +269,9 @@ ESED:=$(STAGING_DIR_HOST)/bin/sed -E -i -e MKHASH:=$(STAGING_DIR_HOST)/bin/mkhash # MKHASH is used in /scripts, so we export it here. export MKHASH +# DOWNLOAD_CHECK_CERTIFICATE is used in /scripts, so we export it here. +DOWNLOAD_CHECK_CERTIFICATE:=$(CONFIG_DOWNLOAD_CHECK_CERTIFICATE) +export DOWNLOAD_CHECK_CERTIFICATE CP:=cp -fpR LN:=ln -sf XARGS:=xargs -r diff --git a/scripts/download.pl b/scripts/download.pl index dd19a52e0..f6fdc1232 100755 --- a/scripts/download.pl +++ b/scripts/download.pl @@ -24,6 +24,8 @@ my $scriptdir = dirname($0); my @mirrors; my $ok; +my $check_certificate = $ENV{DOWNLOAD_CHECK_CERTIFICATE} eq "y"; + $url_filename or $url_filename = $filename; sub localmirrors { @@ -70,21 +72,50 @@ sub hash_cmd() { return undef; } -sub download_cmd($) { +sub download_cmd { my $url = shift; my $have_curl = 0; + my $have_aria2c = 0; + my $filename = shift; + my $additional_mirrors = join(" ", map "$_/$filename", @_); + my @chArray = ('a'..'z', 'A'..'Z', 0..9); + my $rfn = join '', "${filename}_", map{ $chArray[int rand @chArray] } 0..9; if (open CURL, '-|', 'curl', '--version') { if (defined(my $line = readline CURL)) { $have_curl = 1 if $line =~ /^curl /; } close CURL; } + if (open ARIA2C, '-|', 'aria2c', '--version') { + if (defined(my $line = readline ARIA2C)) { + $have_aria2c = 1 if $line =~ /^aria2 /; + } + close ARIA2C; + } - return $have_curl - ? (qw(curl -f --connect-timeout 20 --retry 5 --location --insecure), shellwords($ENV{CURL_OPTIONS} || ''), $url) - : (qw(wget --tries=5 --timeout=20 --no-check-certificate --output-document=-), shellwords($ENV{WGET_OPTIONS} || ''), $url) - ; + if ($have_aria2c) { + @mirrors=(); + return join(" ", "[ -d $ENV{'TMPDIR'}/aria2c ] || mkdir $ENV{'TMPDIR'}/aria2c;", + "touch $ENV{'TMPDIR'}/aria2c/${rfn}_spp;", + qw(aria2c --stderr -c -x2 -s10 -j10 -k1M), $url, $additional_mirrors, + $check_certificate ? () : '--check-certificate=false', + "--server-stat-of=$ENV{'TMPDIR'}/aria2c/${rfn}_spp", + "--server-stat-if=$ENV{'TMPDIR'}/aria2c/${rfn}_spp", + "-d $ENV{'TMPDIR'}/aria2c -o $rfn;", + "cat $ENV{'TMPDIR'}/aria2c/$rfn;", + "rm $ENV{'TMPDIR'}/aria2c/$rfn $ENV{'TMPDIR'}/aria2c/${rfn}_spp"); + } elsif ($have_curl) { + return (qw(curl -f --connect-timeout 20 --retry 5 --location), + $check_certificate ? () : '--insecure', + shellwords($ENV{CURL_OPTIONS} || ''), + $url); + } else { + return (qw(wget --tries=5 --timeout=20 --output-document=-), + $check_certificate ? () : '--no-check-certificate', + shellwords($ENV{WGET_OPTIONS} || ''), + $url); + } } my $hash_cmd = hash_cmd(); @@ -94,6 +125,7 @@ sub download { my $mirror = shift; my $download_filename = shift; + my @additional_mirrors = @_; $mirror =~ s!/$!!; @@ -140,9 +172,9 @@ sub download } }; } else { - my @cmd = download_cmd("$mirror/$download_filename"); + my @cmd = download_cmd("$mirror/$download_filename", $download_filename, @additional_mirrors); print STDERR "+ ".join(" ",@cmd)."\n"; - open(FETCH_FD, '-|', @cmd) or die "Cannot launch curl or wget.\n"; + open(FETCH_FD, '-|', @cmd) or die "Cannot launch aria2c, curl or wget.\n"; $hash_cmd and do { open MD5SUM, "| $hash_cmd > '$target/$filename.hash'" or die "Cannot launch $hash_cmd.\n"; }; @@ -301,9 +333,9 @@ while (!-f "$target/$filename") { my $mirror = shift @mirrors; $mirror or die "No more mirrors to try - giving up.\n"; - download($mirror, $url_filename); + download($mirror, $url_filename, @mirrors); if (!-f "$target/$filename" && $url_filename ne $filename) { - download($mirror, $filename); + download($mirror, $filename, @mirrors); } }