rfc2822.go 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567
  1. package manifest
  2. import (
  3. "bufio"
  4. "fmt"
  5. "io"
  6. "regexp"
  7. "sort"
  8. "strings"
  9. "github.com/docker-library/go-dockerlibrary/architecture"
  10. "github.com/docker-library/go-dockerlibrary/pkg/stripper"
  11. "pault.ag/go/debian/control"
  12. )
  13. var (
  14. GitCommitRegex = regexp.MustCompile(`^[0-9a-f]{1,40}$`)
  15. GitFetchRegex = regexp.MustCompile(`^refs/(heads|tags)/[^*?:]+$`)
  16. )
  17. type Manifest2822 struct {
  18. Global Manifest2822Entry
  19. Entries []Manifest2822Entry
  20. }
  21. type Manifest2822Entry struct {
  22. control.Paragraph
  23. Maintainers []string `delim:"," strip:"\n\r\t "`
  24. Tags []string `delim:"," strip:"\n\r\t "`
  25. SharedTags []string `delim:"," strip:"\n\r\t "`
  26. Architectures []string `delim:"," strip:"\n\r\t "`
  27. GitRepo string
  28. GitFetch string
  29. GitCommit string
  30. Directory string
  31. // architecture-specific versions of the above fields
  32. ArchValues map[string]string
  33. // "ARCH-FIELD: VALUE"
  34. // ala, "s390x-GitCommit: deadbeef"
  35. // (sourced from Paragraph.Values via .SeedArchValues())
  36. Constraints []string `delim:"," strip:"\n\r\t "`
  37. }
  38. var (
  39. DefaultArchitecture = "amd64"
  40. DefaultManifestEntry = Manifest2822Entry{
  41. Architectures: []string{DefaultArchitecture},
  42. GitFetch: "refs/heads/master",
  43. Directory: ".",
  44. }
  45. )
  46. func deepCopyStringsMap(a map[string]string) map[string]string {
  47. b := map[string]string{}
  48. for k, v := range a {
  49. b[k] = v
  50. }
  51. return b
  52. }
  53. func (entry Manifest2822Entry) Clone() Manifest2822Entry {
  54. // SLICES! grr
  55. entry.Maintainers = append([]string{}, entry.Maintainers...)
  56. entry.Tags = append([]string{}, entry.Tags...)
  57. entry.SharedTags = append([]string{}, entry.SharedTags...)
  58. entry.Architectures = append([]string{}, entry.Architectures...)
  59. entry.Constraints = append([]string{}, entry.Constraints...)
  60. // and MAPS, oh my
  61. entry.ArchValues = deepCopyStringsMap(entry.ArchValues)
  62. return entry
  63. }
  64. func (entry *Manifest2822Entry) SeedArchValues() {
  65. for field, val := range entry.Paragraph.Values {
  66. if strings.HasSuffix(field, "-GitRepo") || strings.HasSuffix(field, "-GitFetch") || strings.HasSuffix(field, "-GitCommit") || strings.HasSuffix(field, "-Directory") {
  67. entry.ArchValues[field] = val
  68. }
  69. }
  70. }
  71. const StringSeparator2822 = ", "
  72. func (entry Manifest2822Entry) MaintainersString() string {
  73. return strings.Join(entry.Maintainers, StringSeparator2822)
  74. }
  75. func (entry Manifest2822Entry) TagsString() string {
  76. return strings.Join(entry.Tags, StringSeparator2822)
  77. }
  78. func (entry Manifest2822Entry) SharedTagsString() string {
  79. return strings.Join(entry.SharedTags, StringSeparator2822)
  80. }
  81. func (entry Manifest2822Entry) ArchitecturesString() string {
  82. return strings.Join(entry.Architectures, StringSeparator2822)
  83. }
  84. func (entry Manifest2822Entry) ConstraintsString() string {
  85. return strings.Join(entry.Constraints, StringSeparator2822)
  86. }
  87. // if this method returns "true", then a.Tags and b.Tags can safely be combined (for the purposes of building)
  88. func (a Manifest2822Entry) SameBuildArtifacts(b Manifest2822Entry) bool {
  89. // check xxxarch-GitRepo, etc. fields for sameness first
  90. for _, key := range append(a.archFields(), b.archFields()...) {
  91. if a.ArchValues[key] != b.ArchValues[key] {
  92. return false
  93. }
  94. }
  95. return a.ArchitecturesString() == b.ArchitecturesString() && a.GitRepo == b.GitRepo && a.GitFetch == b.GitFetch && a.GitCommit == b.GitCommit && a.Directory == b.Directory && a.ConstraintsString() == b.ConstraintsString()
  96. }
  97. // returns a list of architecture-specific fields in an Entry
  98. func (entry Manifest2822Entry) archFields() []string {
  99. ret := []string{}
  100. for key, val := range entry.ArchValues {
  101. if val != "" {
  102. ret = append(ret, key)
  103. }
  104. }
  105. sort.Strings(ret)
  106. return ret
  107. }
  108. // returns a new Entry with any of the values that are equal to the values in "defaults" cleared
  109. func (entry Manifest2822Entry) ClearDefaults(defaults Manifest2822Entry) Manifest2822Entry {
  110. entry = entry.Clone() // make absolutely certain we have a deep clone
  111. if entry.MaintainersString() == defaults.MaintainersString() {
  112. entry.Maintainers = nil
  113. }
  114. if entry.TagsString() == defaults.TagsString() {
  115. entry.Tags = nil
  116. }
  117. if entry.SharedTagsString() == defaults.SharedTagsString() {
  118. entry.SharedTags = nil
  119. }
  120. if entry.ArchitecturesString() == defaults.ArchitecturesString() {
  121. entry.Architectures = nil
  122. }
  123. if entry.GitRepo == defaults.GitRepo {
  124. entry.GitRepo = ""
  125. }
  126. if entry.GitFetch == defaults.GitFetch {
  127. entry.GitFetch = ""
  128. }
  129. if entry.GitCommit == defaults.GitCommit {
  130. entry.GitCommit = ""
  131. }
  132. if entry.Directory == defaults.Directory {
  133. entry.Directory = ""
  134. }
  135. for _, key := range defaults.archFields() {
  136. if defaults.ArchValues[key] == entry.ArchValues[key] {
  137. delete(entry.ArchValues, key)
  138. }
  139. }
  140. if entry.ConstraintsString() == defaults.ConstraintsString() {
  141. entry.Constraints = nil
  142. }
  143. return entry
  144. }
  145. func (entry Manifest2822Entry) String() string {
  146. ret := []string{}
  147. if str := entry.MaintainersString(); str != "" {
  148. ret = append(ret, "Maintainers: "+str)
  149. }
  150. if str := entry.TagsString(); str != "" {
  151. ret = append(ret, "Tags: "+str)
  152. }
  153. if str := entry.SharedTagsString(); str != "" {
  154. ret = append(ret, "SharedTags: "+str)
  155. }
  156. if str := entry.ArchitecturesString(); str != "" {
  157. ret = append(ret, "Architectures: "+str)
  158. }
  159. if str := entry.GitRepo; str != "" {
  160. ret = append(ret, "GitRepo: "+str)
  161. }
  162. if str := entry.GitFetch; str != "" {
  163. ret = append(ret, "GitFetch: "+str)
  164. }
  165. if str := entry.GitCommit; str != "" {
  166. ret = append(ret, "GitCommit: "+str)
  167. }
  168. if str := entry.Directory; str != "" {
  169. ret = append(ret, "Directory: "+str)
  170. }
  171. for _, key := range entry.archFields() {
  172. ret = append(ret, key+": "+entry.ArchValues[key])
  173. }
  174. if str := entry.ConstraintsString(); str != "" {
  175. ret = append(ret, "Constraints: "+str)
  176. }
  177. return strings.Join(ret, "\n")
  178. }
  179. func (manifest Manifest2822) String() string {
  180. entries := []Manifest2822Entry{manifest.Global.ClearDefaults(DefaultManifestEntry)}
  181. entries = append(entries, manifest.Entries...)
  182. ret := []string{}
  183. for i, entry := range entries {
  184. if i > 0 {
  185. entry = entry.ClearDefaults(manifest.Global)
  186. }
  187. ret = append(ret, entry.String())
  188. }
  189. return strings.Join(ret, "\n\n")
  190. }
  191. func (entry *Manifest2822Entry) SetGitRepo(arch string, repo string) {
  192. if entry.ArchValues == nil {
  193. entry.ArchValues = map[string]string{}
  194. }
  195. entry.ArchValues[arch+"-GitRepo"] = repo
  196. }
  197. func (entry Manifest2822Entry) ArchGitRepo(arch string) string {
  198. if val, ok := entry.ArchValues[arch+"-GitRepo"]; ok && val != "" {
  199. return val
  200. }
  201. return entry.GitRepo
  202. }
  203. func (entry Manifest2822Entry) ArchGitFetch(arch string) string {
  204. if val, ok := entry.ArchValues[arch+"-GitFetch"]; ok && val != "" {
  205. return val
  206. }
  207. return entry.GitFetch
  208. }
  209. func (entry *Manifest2822Entry) SetGitCommit(arch string, commit string) {
  210. if entry.ArchValues == nil {
  211. entry.ArchValues = map[string]string{}
  212. }
  213. entry.ArchValues[arch+"-GitCommit"] = commit
  214. }
  215. func (entry Manifest2822Entry) ArchGitCommit(arch string) string {
  216. if val, ok := entry.ArchValues[arch+"-GitCommit"]; ok && val != "" {
  217. return val
  218. }
  219. return entry.GitCommit
  220. }
  221. func (entry Manifest2822Entry) ArchDirectory(arch string) string {
  222. if val, ok := entry.ArchValues[arch+"-Directory"]; ok && val != "" {
  223. return val
  224. }
  225. return entry.Directory
  226. }
  227. func (entry Manifest2822Entry) HasTag(tag string) bool {
  228. for _, existingTag := range entry.Tags {
  229. if tag == existingTag {
  230. return true
  231. }
  232. }
  233. return false
  234. }
  235. // HasSharedTag returns true if the given tag exists in entry.SharedTags.
  236. func (entry Manifest2822Entry) HasSharedTag(tag string) bool {
  237. for _, existingTag := range entry.SharedTags {
  238. if tag == existingTag {
  239. return true
  240. }
  241. }
  242. return false
  243. }
  244. // HasArchitecture returns true if the given architecture exists in entry.Architectures
  245. func (entry Manifest2822Entry) HasArchitecture(arch string) bool {
  246. for _, existingArch := range entry.Architectures {
  247. if arch == existingArch {
  248. return true
  249. }
  250. }
  251. return false
  252. }
  253. func (manifest Manifest2822) GetTag(tag string) *Manifest2822Entry {
  254. for _, entry := range manifest.Entries {
  255. if entry.HasTag(tag) {
  256. return &entry
  257. }
  258. }
  259. return nil
  260. }
  261. // GetSharedTag returns a list of entries with the given tag in entry.SharedTags (or the empty list if there are no entries with the given tag).
  262. func (manifest Manifest2822) GetSharedTag(tag string) []Manifest2822Entry {
  263. ret := []Manifest2822Entry{}
  264. for _, entry := range manifest.Entries {
  265. if entry.HasSharedTag(tag) {
  266. ret = append(ret, entry)
  267. }
  268. }
  269. return ret
  270. }
  271. // GetAllSharedTags returns a list of the sum of all SharedTags in all entries of this image manifest (in the order they appear in the file).
  272. func (manifest Manifest2822) GetAllSharedTags() []string {
  273. fakeEntry := Manifest2822Entry{}
  274. for _, entry := range manifest.Entries {
  275. fakeEntry.SharedTags = append(fakeEntry.SharedTags, entry.SharedTags...)
  276. }
  277. fakeEntry.DeduplicateSharedTags()
  278. return fakeEntry.SharedTags
  279. }
  280. type SharedTagGroup struct {
  281. SharedTags []string
  282. Entries []*Manifest2822Entry
  283. }
  284. // GetSharedTagGroups returns a map of shared tag groups to the list of entries they share (as described in https://github.com/docker-library/go-dockerlibrary/pull/2#issuecomment-277853597).
  285. func (manifest Manifest2822) GetSharedTagGroups() []SharedTagGroup {
  286. inter := map[string][]string{}
  287. interOrder := []string{} // order matters, and maps randomize order
  288. interKeySep := ","
  289. for _, sharedTag := range manifest.GetAllSharedTags() {
  290. interKeyParts := []string{}
  291. for _, entry := range manifest.GetSharedTag(sharedTag) {
  292. interKeyParts = append(interKeyParts, entry.Tags[0])
  293. }
  294. interKey := strings.Join(interKeyParts, interKeySep)
  295. if _, ok := inter[interKey]; !ok {
  296. interOrder = append(interOrder, interKey)
  297. }
  298. inter[interKey] = append(inter[interKey], sharedTag)
  299. }
  300. ret := []SharedTagGroup{}
  301. for _, tags := range interOrder {
  302. group := SharedTagGroup{
  303. SharedTags: inter[tags],
  304. Entries: []*Manifest2822Entry{},
  305. }
  306. for _, tag := range strings.Split(tags, interKeySep) {
  307. group.Entries = append(group.Entries, manifest.GetTag(tag))
  308. }
  309. ret = append(ret, group)
  310. }
  311. return ret
  312. }
  313. func (manifest *Manifest2822) AddEntry(entry Manifest2822Entry) error {
  314. if len(entry.Tags) < 1 {
  315. return fmt.Errorf("missing Tags")
  316. }
  317. if entry.GitRepo == "" || entry.GitFetch == "" || entry.GitCommit == "" {
  318. return fmt.Errorf("Tags %q missing one of GitRepo, GitFetch, or GitCommit", entry.TagsString())
  319. }
  320. if invalidMaintainers := entry.InvalidMaintainers(); len(invalidMaintainers) > 0 {
  321. return fmt.Errorf("Tags %q has invalid Maintainers: %q (expected format %q)", entry.TagsString(), strings.Join(invalidMaintainers, ", "), MaintainersFormat)
  322. }
  323. entry.DeduplicateSharedTags()
  324. if invalidArchitectures := entry.InvalidArchitectures(); len(invalidArchitectures) > 0 {
  325. return fmt.Errorf("Tags %q has invalid Architectures: %q", entry.TagsString(), strings.Join(invalidArchitectures, ", "))
  326. }
  327. seenTag := map[string]bool{}
  328. for _, tag := range entry.Tags {
  329. if otherEntry := manifest.GetTag(tag); otherEntry != nil {
  330. return fmt.Errorf("Tags %q includes duplicate tag: %q (duplicated in %q)", entry.TagsString(), tag, otherEntry.TagsString())
  331. }
  332. if otherEntries := manifest.GetSharedTag(tag); len(otherEntries) > 0 {
  333. return fmt.Errorf("Tags %q includes tag conflicting with a shared tag: %q (shared tag in %q)", entry.TagsString(), tag, otherEntries[0].TagsString())
  334. }
  335. if seenTag[tag] {
  336. return fmt.Errorf("Tags %q includes duplicate tag: %q", entry.TagsString(), tag)
  337. }
  338. seenTag[tag] = true
  339. }
  340. for _, tag := range entry.SharedTags {
  341. if otherEntry := manifest.GetTag(tag); otherEntry != nil {
  342. return fmt.Errorf("Tags %q includes conflicting shared tag: %q (duplicated in %q)", entry.TagsString(), tag, otherEntry.TagsString())
  343. }
  344. if seenTag[tag] {
  345. return fmt.Errorf("Tags %q includes duplicate tag: %q (in SharedTags)", entry.TagsString(), tag)
  346. }
  347. seenTag[tag] = true
  348. }
  349. for i, existingEntry := range manifest.Entries {
  350. if existingEntry.SameBuildArtifacts(entry) {
  351. manifest.Entries[i].Tags = append(existingEntry.Tags, entry.Tags...)
  352. manifest.Entries[i].SharedTags = append(existingEntry.SharedTags, entry.SharedTags...)
  353. manifest.Entries[i].DeduplicateSharedTags()
  354. return nil
  355. }
  356. }
  357. manifest.Entries = append(manifest.Entries, entry)
  358. return nil
  359. }
  360. const (
  361. MaintainersNameRegex = `[^\s<>()][^<>()]*`
  362. MaintainersEmailRegex = `[^\s<>()]+`
  363. MaintainersGitHubRegex = `[^\s<>()]+`
  364. MaintainersFormat = `Full Name <contact-email-or-url> (@github-handle) OR Full Name (@github-handle)`
  365. )
  366. var (
  367. MaintainersRegex = regexp.MustCompile(`^(` + MaintainersNameRegex + `)(?:\s+<(` + MaintainersEmailRegex + `)>)?\s+[(]@(` + MaintainersGitHubRegex + `)[)]$`)
  368. )
  369. func (entry Manifest2822Entry) InvalidMaintainers() []string {
  370. invalid := []string{}
  371. for _, maintainer := range entry.Maintainers {
  372. if !MaintainersRegex.MatchString(maintainer) {
  373. invalid = append(invalid, maintainer)
  374. }
  375. }
  376. return invalid
  377. }
  378. func (entry Manifest2822Entry) InvalidArchitectures() []string {
  379. invalid := []string{}
  380. for _, arch := range entry.Architectures {
  381. if _, ok := architecture.SupportedArches[arch]; !ok {
  382. invalid = append(invalid, arch)
  383. }
  384. }
  385. return invalid
  386. }
  387. // DeduplicateSharedTags will remove duplicate values from entry.SharedTags, preserving order.
  388. func (entry *Manifest2822Entry) DeduplicateSharedTags() {
  389. aggregate := []string{}
  390. seen := map[string]bool{}
  391. for _, tag := range entry.SharedTags {
  392. if seen[tag] {
  393. continue
  394. }
  395. seen[tag] = true
  396. aggregate = append(aggregate, tag)
  397. }
  398. entry.SharedTags = aggregate
  399. }
  400. // DeduplicateArchitectures will remove duplicate values from entry.Architectures and sort the result.
  401. func (entry *Manifest2822Entry) DeduplicateArchitectures() {
  402. aggregate := []string{}
  403. seen := map[string]bool{}
  404. for _, arch := range entry.Architectures {
  405. if seen[arch] {
  406. continue
  407. }
  408. seen[arch] = true
  409. aggregate = append(aggregate, arch)
  410. }
  411. sort.Strings(aggregate)
  412. entry.Architectures = aggregate
  413. }
  414. type decoderWrapper struct {
  415. *control.Decoder
  416. }
  417. func (decoder *decoderWrapper) Decode(entry *Manifest2822Entry) error {
  418. // reset Architectures and SharedTags so that they can be either inherited or replaced, not additive
  419. sharedTags := entry.SharedTags
  420. entry.SharedTags = nil
  421. arches := entry.Architectures
  422. entry.Architectures = nil
  423. for {
  424. err := decoder.Decoder.Decode(entry)
  425. if err != nil {
  426. return err
  427. }
  428. // ignore empty paragraphs (blank lines at the start, excess blank lines between paragraphs, excess blank lines at EOF)
  429. if len(entry.Paragraph.Order) == 0 {
  430. continue
  431. }
  432. // if we had no SharedTags or Architectures, restore our "default" (original) values
  433. if len(entry.SharedTags) == 0 {
  434. entry.SharedTags = sharedTags
  435. }
  436. if len(entry.Architectures) == 0 {
  437. entry.Architectures = arches
  438. }
  439. entry.DeduplicateArchitectures()
  440. // pull out any new architecture-specific values from Paragraph.Values
  441. entry.SeedArchValues()
  442. return nil
  443. }
  444. }
  445. func Parse2822(readerIn io.Reader) (*Manifest2822, error) {
  446. reader := stripper.NewCommentStripper(readerIn)
  447. realDecoder, err := control.NewDecoder(bufio.NewReader(reader), nil)
  448. if err != nil {
  449. return nil, err
  450. }
  451. decoder := decoderWrapper{realDecoder}
  452. manifest := Manifest2822{
  453. Global: DefaultManifestEntry.Clone(),
  454. }
  455. if err := decoder.Decode(&manifest.Global); err != nil {
  456. return nil, err
  457. }
  458. if len(manifest.Global.Maintainers) < 1 {
  459. return nil, fmt.Errorf("missing Maintainers")
  460. }
  461. if invalidMaintainers := manifest.Global.InvalidMaintainers(); len(invalidMaintainers) > 0 {
  462. return nil, fmt.Errorf("invalid Maintainers: %q (expected format %q)", strings.Join(invalidMaintainers, ", "), MaintainersFormat)
  463. }
  464. if len(manifest.Global.Tags) > 0 {
  465. return nil, fmt.Errorf("global Tags not permitted")
  466. }
  467. if invalidArchitectures := manifest.Global.InvalidArchitectures(); len(invalidArchitectures) > 0 {
  468. return nil, fmt.Errorf("invalid global Architectures: %q", strings.Join(invalidArchitectures, ", "))
  469. }
  470. for {
  471. entry := manifest.Global.Clone()
  472. err := decoder.Decode(&entry)
  473. if err == io.EOF {
  474. break
  475. }
  476. if err != nil {
  477. return nil, err
  478. }
  479. if !GitFetchRegex.MatchString(entry.GitFetch) {
  480. return nil, fmt.Errorf(`Tags %q has invalid GitFetch (must be "refs/heads/..." or "refs/tags/..."): %q`, entry.TagsString(), entry.GitFetch)
  481. }
  482. if !GitCommitRegex.MatchString(entry.GitCommit) {
  483. return nil, fmt.Errorf(`Tags %q has invalid GitCommit (must be a commit, not a tag or ref): %q`, entry.TagsString(), entry.GitCommit)
  484. }
  485. err = manifest.AddEntry(entry)
  486. if err != nil {
  487. return nil, err
  488. }
  489. }
  490. return &manifest, nil
  491. }