Browse Source

dist/tools: add dlcache.sh

master
Kaspar Schleiser 6 years ago
parent
commit
d69352a237
  1. 1
      .gitignore
  2. 4
      Makefile.include
  3. 11
      dist/tools/dlcache/README.md
  4. 108
      dist/tools/dlcache/dlcache.sh
  5. 1
      makefiles/vars.inc.mk

1
.gitignore vendored

@ -47,3 +47,4 @@ Makefile.local
.ycm_extra_conf.py
.ycm_extra_conf.pyc
__pycache__
.dlcache

4
Makefile.include

@ -16,6 +16,8 @@ APPDIR ?= $(CURDIR)
BINDIRBASE ?= $(APPDIR)/bin
BINDIR ?= $(BINDIRBASE)/$(BOARD)
PKGDIRBASE ?= $(BINDIRBASE)/pkg/$(BOARD)
DLCACHE ?= $(RIOTBASE)/dist/tools/dlcache/dlcache.sh
DLCACHE_DIR ?= $(RIOTBASE)/.dlcache
__DIRECTORY_VARIABLES := \
RIOTBASE \
@ -30,6 +32,7 @@ __DIRECTORY_VARIABLES := \
CCACHE_BASEDIR \
GITCACHE \
PKGDIRBASE \
DLCACHE_DIR \
#
# Make all paths absolute.
@ -45,6 +48,7 @@ override APPDIR := $(abspath $(APPDIR))
override BINDIRBASE := $(abspath $(BINDIRBASE))
override BINDIR := $(abspath $(BINDIR))
override PKGDIRBASE := $(abspath $(PKGDIRBASE))
override DLCACHE_DIR := $(abspath $(DLCACHE_DIR))
# Ensure that all directories are set and don't contain spaces.
ifneq (, $(filter-out 1, $(foreach v,${__DIRECTORY_VARIABLES},$(words ${${v}}))))

11
dist/tools/dlcache/README.md vendored

@ -0,0 +1,11 @@
# Introduction
This script aims to cache http(s)-downloads
## How it works
- if a file with the right name and md5 exists, nothing happens
- if a file with the right name and md5 is in cache, use that
- if a file with the right name but wrong md5 is in cache, redownload
download to cache
- after download, check md5, then copy to target

108
dist/tools/dlcache/dlcache.sh vendored

@ -0,0 +1,108 @@
#!/bin/sh
DLCACHE_DIR=${DLCACHE_DIR:-~/.dlcache}
mkdir -p "$DLCACHE_DIR"
_echo() {
echo "$@" 1>&2
}
if [ "$(uname)" = Darwin ]; then
_locked() {
local lockfile="$1"
shift
while ! shlock -p $$ -f $lockfile; do
sleep 0.2
done
$*
rm $lockfile
}
else
_locked() {
local lockfile="$1"
shift
(
flock -w 600 9 || exit 1
$*
) 9>"$lockfile"
}
fi
if [ "$(uname)" = Darwin ]; then
MD5="md5 -r"
else
MD5=md5sum
fi
calcmd5() {
local file="$1"
local md5="$2"
local file_md5=$(${MD5} "$file" | cut -d\ -f1)
test "$md5" = "$file_md5"
}
downloader() {
if [ -n "$(command -v wget)" ]; then
wget -nv "$1" -O $2
elif [ -n "$(command -v curl)" ]; then
curl -L $1 -o $2
else
_echo "$0: neither wget nor curl available!"
return 1
fi
}
download() {
local url="$1"
local _md5="$2"
local basename_url=$(basename ${url})
local target="${3:-${basename_url}}"
[ -f "$target" ] && {
# if our target file exists, check it's md5.
calcmd5 "$target" "$_md5" && {
_echo "$0: target exists, md5 matches."
exit 0
}
}
local filename="$(basename $url)"
[ -f "$DLCACHE_DIR/$filename" ] && {
# if the file exists in cache, check it's md5 and possibly remove it.
if calcmd5 "$DLCACHE_DIR/$filename" "$_md5"; then
_echo "$0: getting \"$url\" from cache"
else
_echo "$0: \"$DLCACHE_DIR/$filename\" has wrong checksum, re-downloading"
rm "$DLCACHE_DIR/$filename"
fi
}
[ ! -f "$DLCACHE_DIR/$filename" ] && {
_echo "$0: downloading \"$url\"..."
downloader "$url" "$DLCACHE_DIR/$filename" || {
_echo "$0: error downloading $url to $DLCACHE_DIR/$filename!"
exit 1
}
_echo "$0: done downloading \"$url\""
}
calcmd5 "$DLCACHE_DIR/$filename" "$_md5" || {
_echo "$0: checksum mismatch!"
exit 1
}
if [ "$target" = "-" ]; then
cat "$DLCACHE_DIR/$filename"
else
cp "$DLCACHE_DIR/$filename" "$target"
fi
}
_locked "$DLCACHE_DIR/$(basename $1).locked" download "$@"

1
makefiles/vars.inc.mk

@ -68,6 +68,7 @@ export RESET_FLAGS # The parameters to supply to RESET.
export CCACHE_BASEDIR # ccache basedir, allows multiple riot build
# directories to share a ccache directory
export DLCACHE # directory used to cache http downloads
export DOWNLOAD_TO_FILE # Use `$(DOWNLOAD_TO_FILE) $(DESTINATION) $(URL)` to download `$(URL)` to `$(DESTINATION)`.
export DOWNLOAD_TO_STDOUT # Use `$(DOWNLOAD_TO_STDOUT) $(URL)` to download `$(URL)` output `$(URL)` to stdout, e.g. to be piped into `tar xz`.
export UNZIP_HERE # Use `cd $(SOME_FOLDER) && $(UNZIP_HERE) $(SOME_FILE)` to extract the contents of the zip file `$(SOME_FILE)` into `$(SOME_FOLDER)`.

Loading…
Cancel
Save