include/download.mk: handle .gitattributes rules on rawgit method

This fix a long lasting bug/inconsistency with rawgit method and
dl_github_archive script.

The dl_github_archive script works by using the github generated tar.gz
instead of cloning and checkout and the tar.gz is generated by using git
archive command that parse and apply .gitattributes rules.

rawgit command never handled .gitattributes and instead made a simple git
clone and checkout causing the inconsistency.

To fix the inconsistency, add extra steps to call git archive command
and generate an intermediate tar to apply .gitattributes rules.

Also for git log format, Github shorthash length is 8 instead of the
default 7, also apply this locally for each cloned repo to produce
consistent tar.

Signed-off-by: Christian Marangi <[email protected]>

include/download.mk

@@ -215,6 +215,11 @@ define DownloadMethod/github_archive
 endef
 
 # Only intends to be called as a submethod from other DownloadMethod
+#
+# We first clone, checkout and then we generate a tar using the
+# git archive command to apply any rules of .gitattributes
+# To keep consistency with github generated tar archive, we default
+# the short hash to 8 (default is 7). (for git log related usage)
 define DownloadMethod/rawgit
 	echo "Checking out files from the git repository..."; \
 	mkdir -p $(TMP_DIR)/dl && \
@@ -222,11 +227,17 @@ define DownloadMethod/rawgit
 	rm -rf $(SUBDIR) && \
 	[ \! -d $(SUBDIR) ] && \
 	git clone $(OPTS) $(URL) $(SUBDIR) && \
-	(cd $(SUBDIR) && git checkout $(VERSION) && \
-	$(if $(filter skip,$(SUBMODULES)),true,git submodule update --init --recursive -- $(SUBMODULES))) && \
-	echo "Packing checkout..." && \
+	(cd $(SUBDIR) && git checkout $(VERSION)) && \
 	export TAR_TIMESTAMP=`cd $(SUBDIR) && git log -1 --format='@%ct'` && \
-	rm -rf $(SUBDIR)/.git && \
+	echo "Generating formal git archive (apply .gitattributes rules)" && \
+	(cd $(SUBDIR) && git config core.abbrev 8 && \
+	git archive --format=tar HEAD --output=../$(SUBDIR).tar.git) && \
+	$(if $(filter skip,$(SUBMODULES)),true,$(TAR) --ignore-failed-read -C $(SUBDIR) -f $(SUBDIR).tar.git -r .git .gitmodules 2>/dev/null) && \
+	rm -rf $(SUBDIR) && mkdir $(SUBDIR) && \
+	$(TAR) -C $(SUBDIR) -xf $(SUBDIR).tar.git && \
+	(cd $(SUBDIR) && $(if $(filter skip,$(SUBMODULES)),true,git submodule update --init --recursive -- $(SUBMODULES) && \
+	rm -rf .git .gitmodules)) && \
+	echo "Packing checkout..." && \
 	$(call dl_tar_pack,$(TMP_DIR)/dl/$(FILE),$(SUBDIR)) && \
 	mv $(TMP_DIR)/dl/$(FILE) $(DL_DIR)/ && \
 	rm -rf $(SUBDIR);