Browse Source

Merge pull request #6643 from kaspar030/add_http_dl_cache

dist/tools: add http download with caching
master
Kaspar Schleiser 5 years ago committed by GitHub
parent
commit
ed9899ad57
  1. 1
      .gitignore
  2. 4
      Makefile.include
  3. 3
      boards/x86-multiboot-common/Makefile.include
  4. 11
      dist/tools/dlcache/README.md
  5. 108
      dist/tools/dlcache/dlcache.sh
  6. 1
      makefiles/vars.inc.mk
  7. 3
      pkg/nordic_softdevice_ble/Makefile
  8. 3
      pkg/tlsf/Makefile

1
.gitignore vendored

@ -47,3 +47,4 @@ Makefile.local
.ycm_extra_conf.py
.ycm_extra_conf.pyc
__pycache__
.dlcache

4
Makefile.include

@ -16,6 +16,8 @@ APPDIR ?= $(CURDIR)
BINDIRBASE ?= $(APPDIR)/bin
BINDIR ?= $(BINDIRBASE)/$(BOARD)
PKGDIRBASE ?= $(BINDIRBASE)/pkg/$(BOARD)
DLCACHE ?= $(RIOTBASE)/dist/tools/dlcache/dlcache.sh
DLCACHE_DIR ?= $(RIOTBASE)/.dlcache
__DIRECTORY_VARIABLES := \
RIOTBASE \
@ -30,6 +32,7 @@ __DIRECTORY_VARIABLES := \
CCACHE_BASEDIR \
GITCACHE \
PKGDIRBASE \
DLCACHE_DIR \
#
# Make all paths absolute.
@ -45,6 +48,7 @@ override APPDIR := $(abspath $(APPDIR))
override BINDIRBASE := $(abspath $(BINDIRBASE))
override BINDIR := $(abspath $(BINDIR))
override PKGDIRBASE := $(abspath $(PKGDIRBASE))
override DLCACHE_DIR := $(abspath $(DLCACHE_DIR))
# Ensure that all directories are set and don't contain spaces.
ifneq (, $(filter-out 1, $(foreach v,${__DIRECTORY_VARIABLES},$(words ${${v}}))))

3
boards/x86-multiboot-common/Makefile.include

@ -5,11 +5,12 @@ endif
ifneq (0, $(shell test -e "$(NEWLIB_BASE)/lib/libc.a" && echo $$?))
NEWLIB_PRECOMPILED_NAME := i586-newlib_2.2.0.20150623_tlsf-3337.tar.bz2
NEWLIB_PRECOMPILED := http://download.riot-os.org/$(NEWLIB_PRECOMPILED_NAME)
NEWLIB_PRECOMPILED_MD5 := 8090a768cf0294b259fe7ad904d085c7
$(warning Precompiled newlib is missing in $(NEWLIB_BASE))
$(warning Downloading from $(NEWLIB_PRECOMPILED))
$(shell cd $(RIOTBASE) && $(DOWNLOAD_TO_STDOUT) "$(NEWLIB_PRECOMPILED)" | tar xj)
$(shell cd $(RIOTBASE) && $(DLCACHE) "$(NEWLIB_PRECOMPILED)" 8090a768cf0294b259fe7ad904d085c7 - | tar xj)
endif
ifeq (,$(BUILD_INCLUDE_BASE))

11
dist/tools/dlcache/README.md vendored

@ -0,0 +1,11 @@
# Introduction
This script aims to cache http(s)-downloads
## How it works
- if a file with the right name and md5 exists, nothing happens
- if a file with the right name and md5 is in cache, use that
- if a file with the right name but wrong md5 is in cache, redownload
download to cache
- after download, check md5, then copy to target

108
dist/tools/dlcache/dlcache.sh vendored

@ -0,0 +1,108 @@
#!/bin/sh
DLCACHE_DIR=${DLCACHE_DIR:-~/.dlcache}
mkdir -p "$DLCACHE_DIR"
_echo() {
echo "$@" 1>&2
}
if [ "$(uname)" = Darwin ]; then
_locked() {
local lockfile="$1"
shift
while ! shlock -p $$ -f $lockfile; do
sleep 0.2
done
$*
rm $lockfile
}
else
_locked() {
local lockfile="$1"
shift
(
flock -w 600 9 || exit 1
$*
) 9>"$lockfile"
}
fi
if [ "$(uname)" = Darwin ]; then
MD5="md5 -r"
else
MD5=md5sum
fi
calcmd5() {
local file="$1"
local md5="$2"
local file_md5=$(${MD5} "$file" | cut -d\ -f1)
test "$md5" = "$file_md5"
}
downloader() {
if [ -n "$(command -v wget)" ]; then
wget -nv "$1" -O $2
elif [ -n "$(command -v curl)" ]; then
curl -L $1 -o $2
else
_echo "$0: neither wget nor curl available!"
return 1
fi
}
download() {
local url="$1"
local _md5="$2"
local basename_url=$(basename ${url})
local target="${3:-${basename_url}}"
[ -f "$target" ] && {
# if our target file exists, check it's md5.
calcmd5 "$target" "$_md5" && {
_echo "$0: target exists, md5 matches."
exit 0
}
}
local filename="$(basename $url)"
[ -f "$DLCACHE_DIR/$filename" ] && {
# if the file exists in cache, check it's md5 and possibly remove it.
if calcmd5 "$DLCACHE_DIR/$filename" "$_md5"; then
_echo "$0: getting \"$url\" from cache"
else
_echo "$0: \"$DLCACHE_DIR/$filename\" has wrong checksum, re-downloading"
rm "$DLCACHE_DIR/$filename"
fi
}
[ ! -f "$DLCACHE_DIR/$filename" ] && {
_echo "$0: downloading \"$url\"..."
downloader "$url" "$DLCACHE_DIR/$filename" || {
_echo "$0: error downloading $url to $DLCACHE_DIR/$filename!"
exit 1
}
_echo "$0: done downloading \"$url\""
}
calcmd5 "$DLCACHE_DIR/$filename" "$_md5" || {
_echo "$0: checksum mismatch!"
exit 1
}
if [ "$target" = "-" ]; then
cat "$DLCACHE_DIR/$filename"
else
cp "$DLCACHE_DIR/$filename" "$target"
fi
}
_locked "$DLCACHE_DIR/$(basename $1).locked" download "$@"

1
makefiles/vars.inc.mk

@ -68,6 +68,7 @@ export RESET_FLAGS # The parameters to supply to RESET.
export CCACHE_BASEDIR # ccache basedir, allows multiple riot build
# directories to share a ccache directory
export DLCACHE # directory used to cache http downloads
export DOWNLOAD_TO_FILE # Use `$(DOWNLOAD_TO_FILE) $(DESTINATION) $(URL)` to download `$(URL)` to `$(DESTINATION)`.
export DOWNLOAD_TO_STDOUT # Use `$(DOWNLOAD_TO_STDOUT) $(URL)` to download `$(URL)` output `$(URL)` to stdout, e.g. to be piped into `tar xz`.
export UNZIP_HERE # Use `cd $(SOME_FOLDER) && $(UNZIP_HERE) $(SOME_FILE)` to extract the contents of the zip file `$(SOME_FILE)` into `$(SOME_FOLDER)`.

3
pkg/nordic_softdevice_ble/Makefile

@ -2,6 +2,7 @@ PKG_NAME = nordic_softdevice_ble
PKG_VERSION = 3288530
PKG_FILE = nrf5_iot_sdk_$(PKG_VERSION).zip
PKG_URL = https://developer.nordicsemi.com/nRF5_IoT_SDK/nRF5_IoT_SDK_v0.9.x/$(PKG_FILE)
PKG_MD5 = 291c875df69c51e22491e6112128bd9c
PKG_LICENSE = nordic-bsd
PKG_DIR=$(CURDIR)
PKG_BUILDDIR=$(PKGDIRBASE)/$(PKG_NAME)
@ -41,7 +42,7 @@ $(PKG_SRCDIR)/.extracted: $(PKG_BUILDDIR)/$(PKG_FILE)
$(PKG_BUILDDIR)/$(PKG_FILE):
@mkdir -p $(@D)
$(Q)$(DOWNLOAD_TO_FILE) $@ $(PKG_URL)
$(Q)$(DLCACHE) $(PKG_URL) $(PKG_MD5) $@
clean::
rm -rf $(PKG_SRCDIR)/

3
pkg/tlsf/Makefile

@ -2,6 +2,7 @@ PKG_NAME = tlsf
PKG_VERSION = 3.0
PKG_FILE = tlsf-$(PKG_VERSION).zip
PKG_URL = http://download.riot-os.org/$(PKG_FILE)
PKG_MD5 = 176965d58af525347e582e1630ea9604
PKG_LICENSE = PD
PKG_DIR=$(CURDIR)
PKG_BUILDDIR=$(PKGDIRBASE)/$(PKG_NAME)
@ -22,7 +23,7 @@ $(PKG_SRCDIR)/Makefile: $(PKG_BUILDDIR)/$(PKG_FILE) $(CURDIR)/patch.txt
$(PKG_BUILDDIR)/$(PKG_FILE):
@mkdir -p $(@D)
$(Q)$(DOWNLOAD_TO_FILE) $@ $(PKG_URL)
$(Q)$(DLCACHE) $(PKG_URL) $(PKG_MD5) $@
clean::
rm -rf $(PKG_SRCDIR)/

Loading…
Cancel
Save