buildtest: retry failed builds once

On Travis CI often builds fail spuriously.

This PR lets `make buildtest` retry the build once.

Makefile.buildtests

@@ -22,10 +22,10 @@ endif
 
 buildtest:
 	@if [ -z "$${JENKINS_URL}" ] && tput colors 2>&1 > /dev/null; then \
-		GREEN='\033[1;32m'; RED='\033[1;31m'; RESET='\033[0m'; \
+		GREEN='\033[1;32m'; RED='\033[1;31m'; PURPLE='\033[1;35m'; RESET='\033[0m'; \
 		ECHO='/bin/echo -e'; \
 	else \
-		GREEN=''; RED=''; RESET=''; \
+		GREEN=''; RED=''; PURPLE=''; RESET=''; \
 		ECHO='/bin/echo'; \
 	fi; \
 	\
@@ -40,34 +40,39 @@ buildtest:
 		BOARDS=$$(echo \ $${BOARDS}\  | sed -e 's/ '$${BOARD}' / /'); \
 	done; \
 	\
+	BUILDTESTOK=true; \
 	rm -rf "$$BINDIRBASE"; \
 	for BOARD in $${BOARDS}; do \
 		RIOTNOLINK=$$(case ' $(BOARD_INSUFFICIENT_RAM) ' in *" $${BOARD} "*) echo 1; esac); \
 		$${ECHO} -n "Building for $${BOARD} "; \
 		[ -n "$${RIOTNOLINK}" ] && $${ECHO} -n "(no linking) "; \
-		$${ECHO} -n ".. "; \
-		LOG=$$(env -i \
-			HOME=$${HOME} \
-			PATH=$${PATH} \
-			BOARD=$${BOARD} \
-			RIOTBASE=$${RIOTBASE} \
-			RIOTBOARD=$${RIOTBOARD} \
-			RIOTCPU=$${RIOTCPU} \
-			BINDIRBASE=$${BINDIRBASE} \
-			RIOTNOLINK=$${RIOTNOLINK} \
-			RIOT_VERSION=$${RIOT_VERSION} \
-			$(MAKE) -j$(NPROC) 2>&1 >/dev/null) ; \
-		if [ "$${?}" = "0" ]; then \
-			$${ECHO} "$${GREEN}success$${RESET}"; \
-		else \
-			$${ECHO} "$${RED}failed$${RESET}"; \
-			echo "$${LOG}" | grep -v -E '^make(\[[[:digit:]]])?:'; \
-			BUILDTESTFAILED=1; \
-		fi; \
+		for NTH_TRY in 1 2; do \
+			$${ECHO} -n ".. "; \
+			LOG=$$(env -i \
+					HOME=$${HOME} \
+					PATH=$${PATH} \
+					BOARD=$${BOARD} \
+					RIOTBASE=$${RIOTBASE} \
+					RIOTBOARD=$${RIOTBOARD} \
+					RIOTCPU=$${RIOTCPU} \
+					BINDIRBASE=$${BINDIRBASE} \
+					RIOTNOLINK=$${RIOTNOLINK} \
+					RIOT_VERSION=$${RIOT_VERSION} \
+					$(MAKE) -j$(NPROC) 2>&1 >/dev/null) ; \
+			if [ "$${?}" = "0" ]; then \
+				$${ECHO} "$${GREEN}success$${RESET}"; \
+			elif [ -n "$${RIOT_DO_RETRY}" ] && $${BUILDTESTOK} && [ $${NTH_TRY} = 1 ]; then \
+				$${ECHO} -n "$${PURPLE}retrying$${RESET} "; \
+				continue; \
+			else \
+				$${ECHO} "$${RED}failed$${RESET}"; \
+				echo "$${LOG}" | grep -v -E '^make(\[[[:digit:]]])?:'; \
+				BUILDTESTOK=false; \
+			fi; \
+			break; \
+		done; \
 	done; \
-	if [ "$${BUILDTESTFAILED}" = "1" ]; then \
-		exit 1; \
-	fi
+	$${BUILDTESTOK}
 
 objsize:
 	@case "${SORTROW}" in \

dist/tools/compile_test/compile_test.py

@@ -53,10 +53,10 @@ def get_lines(readline, prefix):
             continue
 
         result = result[len(prefix):].rstrip().split(' .. ')[::-1]
-        if len(result) == 2:
+        if (len(result) == 2) or (len(result) == 3 and 'retrying' in result[1]):
             stdout.write('.')
             stdout.flush()
-            yield result
+            yield (' .. '.join(result[:-1]), result[-1])
 
 for folder in ('examples', 'tests'):
     print('Building all applications in: \033[1;34m{}\033[0m'.format(folder))
@@ -65,21 +65,31 @@ for folder in ('examples', 'tests'):
     applications = filter(lambda app: is_tracked(join(riotbase, folder, app)), applications)
     applications = sorted(applications)
 
+    subprocess_env = environ.copy()
+    subprocess_env['RIOT_DO_RETRY'] = '1'
+
     for nth, application in enumerate(applications, 1):
         stdout.write('\tBuilding application: \033[1;34m{}\033[0m ({}/{}) '.format(application, nth, len(applications)))
         stdout.flush()
         try:
             subprocess = Popen(('make', 'buildtest'),
                                bufsize=1, stdin=null, stdout=PIPE, stderr=null,
-                               cwd=join(riotbase, folder, application))
+                               cwd=join(riotbase, folder, application),
+                               env=subprocess_env)
 
             lines = get_lines(subprocess.stdout.readline, 'Building for ')
             lines = groupby(sorted(lines), lambda (outcome, board): outcome)
+
+            print()
             for group, results in lines:
-                print('\n\t\t{}: {}'.format(group, ', '.join(sorted(board for outcome, board in results))))
+                print('\t\t{}: {}'.format(group, ', '.join(sorted(board for outcome, board in results))))
 
             returncode = subprocess.wait()
-            (failed if returncode else success).append(application)
+            if returncode == 0:
+                success.append(application)
+            else:
+                failed.append(application)
+                subprocess_env.pop('RIOT_DO_RETRY', None)
         except Exception, e:
             print('\n\t\tException: {}'.format(e))
             exceptions.append(application)