Merge pull request #599 from infosiftr/schema-version-2

Add proper support for schemaVersion 2 (including manifest lists) in generate-tag-details.pl

.template-helpers/generate-tag-details.pl

@@ -8,8 +8,21 @@ use Mojo::UserAgent;
 
 die 'no images specified' unless @ARGV;
 
+my $mediaManifestList = 'application/vnd.docker.distribution.manifest.list.v2+json';
+my $mediaManifestV2 = 'application/vnd.docker.distribution.manifest.v2+json';
+my $mediaManifestV1 = 'application/vnd.docker.distribution.manifest.v1+json';
+
 my $ua = Mojo::UserAgent->new->max_redirects(10);
-$ua->transactor->name('Docker');
+$ua->transactor->name(join ' ',
+	# https://github.com/docker/docker/blob/v1.11.2/dockerversion/useragent.go#L13-L34
+	'docker/1.11.2',
+	'go/1.6.2',
+	'git-commit/v1.11.2',
+	'kernel/4.4.11',
+	'os/linux',
+	'arch/amd64',
+	# BOGUS USER AGENTS FOR THE BOGUS USER AGENT THRONE
+);
 
 my $maxRetries = 3;
 sub ua_req {
@@ -19,7 +32,7 @@ sub ua_req {
 	do {
 		--$tries;
 		$tx = $ua->$method(@_);
-		return $tx if $tx->success;
+		return $tx if $tx->success or $tx->res->code == 401 or $tx->res->code == 404;
 	} while ($tries > 0);
 	return $tx;
 }
@@ -43,45 +56,73 @@ sub split_image_name {
 	die "unrecognized image name format in: $image";
 }
 
-sub get_token {
+sub registry_req {
+	my $method = shift;
 	my $repo = shift;
+	my $url = shift;
+	my %extHeaders = @_;
+
 	state %tokens;
-	return $tokens{$repo} if $tokens{$repo};
-	my $realmTx = $ua->get("https://registry-1.docker.io/v2/$repo/tags/list");
-	my $auth = $realmTx->res->headers->www_authenticate;
-	die "unexpected WWW-Authenticate header: $auth" unless $auth =~ m{ ^ Bearer \s+ (\S.*) $ }x;
-	my $realm = $1;
-	my $url = Mojo::URL->new;
-	while ($realm =~ m{
-		# key="val",
-		([^=]+)
-		=
-		"([^"]+)"
-		,?
-	}xg) {
-		my ($key, $val) = ($1, $2);
-		if ($key eq 'realm') {
-			$url->base(Mojo::URL->new($val));
-		} else {
-			$url->query->append($key => $val);
+
+	$url = "https://registry-1.docker.io/v2/$repo/$url";
+
+	for (;;) {
+		my %headers = (
+			%extHeaders,
+		);
+
+		if (my $token = $tokens{$repo}) {
+			$headers{Authorization} = "Bearer $token";
 		}
+
+		my $tx = ua_req($method => $url => \%headers);
+
+		if ($tx->res->code == 401) {
+			my $auth = $tx->res->headers->www_authenticate;
+			die "unexpected WWW-Authenticate header: $auth" unless $auth =~ m{ ^ Bearer \s+ (\S.*) $ }x;
+			my $realm = $1;
+			my $authUrl = Mojo::URL->new;
+			while ($realm =~ m{
+				# key="val",
+				([^=]+)
+				=
+				"([^"]+)"
+				,?
+			}xg) {
+				my ($key, $val) = ($1, $2);
+				if ($key eq 'realm') {
+					$authUrl->base(Mojo::URL->new($val));
+				} else {
+					$authUrl->query->append($key => $val);
+				}
+			}
+			$authUrl = $authUrl->to_abs;
+			my $tokenTx = ua_req(get => $authUrl);
+			die "failed to fetch token for $repo" unless $tokenTx->success;
+			$tokens{$repo} = $tokenTx->res->json->{token};
+			next;
+		}
+
+		return $tx;
 	}
-	$url = $url->to_abs;
-	my $tokenTx = ua_req(get => $url);
-	die "failed to fetch token for $repo" unless $tokenTx->success;
-	return $tokens{$repo} = $tokenTx->res->json->{token};
 }
 
 sub get_manifest {
 	my ($repo, $tag) = @_;
+
 	my $image = "$repo:$tag";
 	state %manifests;
 	return $manifests{$image} if $manifests{$image};
 
-	my $token = get_token($repo);
-	my $authorizationHeader = { Authorization => "Bearer $token" };
-
-	my $manifestTx = ua_req(get => "https://registry-1.docker.io/v2/$repo/manifests/$tag" => $authorizationHeader);
+	my $manifestTx = registry_req(get => $repo => "manifests/$tag" => (
+			# prefer a "version 2" manifest
+			# https://docs.docker.com/registry/spec/manifest-v2-2/
+			Accept => [
+				$mediaManifestList,
+				$mediaManifestV2,
+				$mediaManifestV1,
+			],
+		));
 	return () if $manifestTx->res->code == 404; # tag doesn't exist
 	die "failed to get manifest for $image" unless $manifestTx->success;
 	return (
@@ -90,16 +131,34 @@ sub get_manifest {
 	);
 }
 
+sub blob_req {
+	my $method = shift;
+	my $repo = shift;
+	my $blob = shift;
+	my %extHeaders = @_;
+	return registry_req($method => $repo => "blobs/$blob" => %extHeaders);
+}
+
+sub get_blob_json {
+	my ($repo, $blob) = @_;
+
+	my $key = $repo . '@' . $blob;
+	state %blobs;
+	return $blobs{$key} if $blobs{$key};
+
+	my $tx = blob_req(get => ($repo, $blob) => ());
+	die "failed to get blob data for $key" unless $tx->success;
+	return $blobs{$key} = $tx->res->json;
+}
+
 sub get_blob_headers {
 	my ($repo, $blob) = @_;
+
 	my $key = $repo . '@' . $blob;
 	state %headers;
 	return $headers{$key} if $headers{$key};
 
-	my $token = get_token($repo);
-	my $authorizationHeader = { Authorization => "Bearer $token" };
-
-	my $headersTx = ua_req(head => "https://registry-1.docker.io/v2/$repo/blobs/$blob" => $authorizationHeader);
+	my $headersTx = blob_req(head => ($repo, $blob) => ());
 	die "failed to get headers for $key" unless $headersTx->success;
 	return $headers{$key} = $headersTx->res->headers;
 }
@@ -122,6 +181,181 @@ sub get_layer_data {
 	return $layers{$id} = $data;
 }
 
+sub parse_manifest_v1_data {
+	my ($repo, $manifest) = @_;
+
+	my $data = {
+		manifestVersion => $mediaManifestV1,
+		manifest => $manifest,
+		imageId => undef,
+		platform => {},
+		dockerVersion => undef,
+		entrypoint => undef,
+		defaultCommand => undef,
+		layers => [],
+		commands => [],
+	};
+
+	my %seenBlob;
+	for my $fsLayer (@{ $manifest->{fsLayers} // [] }) {
+		my $blob = $fsLayer->{blobSum};
+		next unless $blob;
+
+		next if $seenBlob{$blob};
+		$seenBlob{$blob} = 1;
+
+		push @{ $data->{layers} }, {
+			digest => $blob,
+		};
+	}
+
+	for my $history (@{ $manifest->{history} // [] }) {
+		next unless $history->{v1Compatibility};
+
+		my $v1 = Mojo::Util::encode('UTF-8', $history->{v1Compatibility});
+
+		my $json = Mojo::JSON::decode_json($v1);
+
+		$data->{dockerVersion} //= $json->{docker_version};
+		$data->{platform}{os} //= $json->{os};
+		$data->{platform}{architecture} //= $json->{architecture};
+		$data->{entrypoint} //= $json->{config}{Entrypoint};
+		$data->{defaultCommand} //= $json->{config}{Cmd};
+		$data->{imageId} //= $json->{id};
+
+		# "history" in v1 is in reverse order (hence "unshift" instead of "push")
+		unshift @{ $data->{commands} }, {
+			created => $json->{created},
+			command => $json->{container_config}{Cmd},
+		};
+	}
+
+	return $data;
+}
+
+sub parse_manifest_v2_data {
+	my ($repo, $manifest) = @_;
+
+	my $configDigest = $manifest->{config}{digest};
+	my $config = get_blob_json($repo, $configDigest);
+
+	return {
+		manifestVersion => $mediaManifestV2,
+		manifest => $manifest,
+		imageId => $configDigest,
+		config => $config,
+		platform => {
+			os => $config->{os},
+			architecture => $config->{architecture},
+		},
+		dockerVersion => $config->{docker_version},
+		entrypoint => $config->{config}{Entrypoint},
+		defaultCommand => $config->{config}{Cmd},
+		layers => $manifest->{layers} // [],
+		commands => $config->{history} // [],
+	};
+}
+
+sub get_image_data {
+	my ($image) = @_;
+
+	my ($repo, $tag) = split_image_name($image);
+
+	my ($digest, $manifest) = get_manifest($repo, $tag);
+
+	unless (defined $digest && defined $manifest) {
+		# tag must not exist!
+		return;
+	}
+
+	my $data = {
+		repo => $repo,
+		tag => $tag,
+		digest => $digest,
+		images => [],
+	};
+
+	if ($manifest->{schemaVersion} eq '1') {
+		# https://docs.docker.com/registry/spec/manifest-v2-1/
+		push @{$data->{images}}, parse_manifest_v1_data($repo, $manifest);
+	}
+	elsif ($manifest->{schemaVersion} eq '2') {
+		# https://docs.docker.com/registry/spec/manifest-v2-2/
+		if ($manifest->{mediaType} eq $mediaManifestV2) {
+			push @{$data->{images}}, parse_manifest_v2_data($repo, $manifest);
+		}
+		elsif ($manifest->{mediaType} eq $mediaManifestList) {
+			$data->{manifest} = $manifest;
+			$data->{manifestVersion} = $manifest->{mediaType};
+
+			for my $sub (@{ $manifest->{manifests} // [] }) {
+				my $digest = $sub->{digest};
+				die "sub-manifest missing digest!" unless $digest;
+
+				my $subManifest = get_manifest($repo, $digest);
+				die "manifest $digest does not exist!" unless defined $subManifest;
+
+				my $subData;
+				if ($sub->{mediaType} eq $mediaManifestV1) {
+					$subData = parse_manifest_v1_data($repo, $subManifest);
+				}
+				elsif ($sub->{mediaType} eq $mediaManifestV2) {
+					$subData = parse_manifest_v2_data($repo, $subManifest);
+				}
+				else {
+					die "unknown mediaType $manifest->{mediaType} for $digest";
+				}
+
+				$subData->{digest} = $digest;
+				$subData->{platform} = $sub->{platform};
+
+				push @{$data->{images}}, $subData;
+			}
+		}
+		else {
+			die "unknown mediaType $manifest->{mediaType} for schemaVersion 2";
+		}
+	}
+	else {
+		die "unknown schemaVersion: $manifest->{schemaVersion}";
+	}
+
+	for my $image (@{ $data->{images} }) {
+		$image->{platform} //= {};
+
+		$image->{layers} //= [];
+		$image->{size} = 0;
+		for my $layer (@{ $image->{layers} }) {
+			my $headers = get_blob_headers($repo, $layer->{digest});
+			$layer->{size} //= $headers->content_length;
+			$layer->{mediaType} //= $headers->content_type;
+			$layer->{lastModified} //= $headers->last_modified;
+			$image->{size} += $layer->{size};
+		}
+
+		$image->{commands} //= [];
+		for my $command (@{ $image->{commands} }) {
+			$command->{command} //= [ $command->{created_by} ];
+			$command->{dockerfile} //= cmd_to_dockerfile($command->{command});
+		}
+	}
+
+	return $data;
+}
+
+sub platform_string {
+	my $platform = shift;
+	return (
+		($platform->{os} // 'linux')
+		. (defined $platform->{'os.version'} ? ' version ' . $platform->{'os.version'} : '')
+		. (defined $platform->{'os.features'} ? ' ft. ' . join(', ', @{ $platform->{'os.features'} }) : '')
+		. '; '
+		. ($platform->{architecture} // 'amd64')
+		. (defined $platform->{variant} ? ' variant ' . $platform->{variant} : '')
+		. (defined $platform->{features} ? ' ft. ' . join(', ', @{ $platform->{features} }) : '')
+	);
+}
+
 sub cmd_to_dockerfile {
 	my ($cmd) = @_;
 
@@ -204,61 +438,66 @@ sub date {
 
 while (my $image = shift) {
 	print "\n";
+
 	say '## `' . $image . '`';
-	my ($repo, $tag) = split_image_name($image);
 
-	my ($digest, $manifest) = get_manifest($repo, $tag);
+	my $data = get_image_data($image);
 
-	unless (defined $digest && defined $manifest) {
+	unless ($data) {
 		# tag must not exist yet!
 		say "\n", '**does not exist** (yet?)';
 		next;
 	}
 
+	my $repo = $data->{repo};
+	$repo =~ s!^library/!!;
+
 	print "\n";
 	say '```console';
-	say '$ docker pull ' . $repo . '@' . $digest;
+	say '$ docker pull ' . $repo . '@' . $data->{digest};
 	say '```';
 
-	my %parentChild;
-	my %totals = (
-		virtual_size => 0,
-		blob_content_length => 0,
-	);
-	for my $i (0 .. $#{ $manifest->{fsLayers} }) {
-		my $v1 = Mojo::Util::encode 'UTF-8', $manifest->{history}[$i]{v1Compatibility};
-		my $data = get_layer_data(
-			$repo, undef,
-			$manifest->{fsLayers}[$i]{blobSum},
-			Mojo::JSON::decode_json($v1),
-		);
-		$parentChild{$data->{parent} // ''} = $data->{id};
-		$totals{$_} += $data->{$_} for keys %totals;
-	}
-	print "\n";
-	say "-\t" . 'Total Virtual Size: ' . size($totals{virtual_size}) if $totals{virtual_size};
-	say "-\t" . 'Total v2 Content-Length: ' . size($totals{blob_content_length});
 	print "\n";
-	say '### Layers (' . scalar(keys %parentChild) . ')';
-	my $cur = $parentChild{''};
-	while ($cur) {
+	say '- Manifest MIME: `' . $data->{manifestVersion} . '`' if $data->{manifestVersion};
+	say '- Platforms:';
+	for my $imageData (@{ $data->{images} }) {
+		say '  - ' . platform_string($imageData->{platform});
+	}
+
+	for my $imageData (@{ $data->{images} }) {
 		print "\n";
-		say '#### `' . $cur . '`';
-		my $data = get_layer_data($repo, $cur);
-		if ($data->{container_command}) {
+		say '### `' . $image . '` - ' . platform_string($imageData->{platform});
+
+		if ($imageData->{digest}) {
 			print "\n";
-			say '```dockerfile';
-			say cmd_to_dockerfile($data->{container_command});
+			say '```console';
+			say '$ docker pull ' . $repo . '@' . $imageData->{digest};
 			say '```';
 		}
+
+		print "\n";
+		say '- Docker Version: ' . $imageData->{dockerVersion} if $imageData->{dockerVersion};
+		say '- Manifest MIME: `' . $imageData->{manifestVersion} . '`' if $imageData->{manifestVersion};
+		say '- Total Size: **' . size($imageData->{size}) . '**  ';
+		say '  (compressed transfer size, not on-disk size)';
+		say '- Image ID: `' . $imageData->{imageId} . '`' if $imageData->{imageId};
+		say '- Entrypoint: `' . Mojo::JSON::encode_json($imageData->{entrypoint}) . '`' if $imageData->{entrypoint} && @{ $imageData->{entrypoint} };
+		say '- Default Command: `' . Mojo::JSON::encode_json($imageData->{defaultCommand}) . '`' if $imageData->{defaultCommand};
+
+		print "\n";
+		say '```dockerfile';
+		for my $command (@{ $imageData->{commands} }) {
+			say '# ' . date($command->{created});
+			say $command->{dockerfile};
+		}
+		say '```';
+
 		print "\n";
-		say "-\t" . 'Created: ' . date($data->{created}) if $data->{created};
-		say "-\t" . 'Parent Layer: `' . $data->{parent} . '`' if $data->{parent};
-		say "-\t" . 'Docker Version: ' . $data->{docker_version} if $data->{docker_version};
-		say "-\t" . 'Virtual Size: ' . size($data->{virtual_size}) if $totals{virtual_size};
-		say "-\t" . 'v2 Blob: `' . $data->{blob} . '`';
-		say "-\t" . 'v2 Content-Length: ' . size($data->{blob_content_length});
-		say "-\t" . 'v2 Last-Modified: ' . date($data->{blob_last_modified}) if $data->{blob_last_modified};
-		$cur = $parentChild{$cur};
+		say '- Layers:';
+		for my $layer (@{ $imageData->{layers} }) {
+			say '  - `' . $layer->{digest} . '`  ';
+			say '    Last Modified: ' . date($layer->{lastModified}) . '  ';
+			say '    Size: ' . size($layer->{size});
+		}
 	}
 }