rfc2822.go 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582
  1. package manifest
  2. import (
  3. "bufio"
  4. "fmt"
  5. "io"
  6. "regexp"
  7. "sort"
  8. "strings"
  9. "github.com/docker-library/go-dockerlibrary/architecture"
  10. "github.com/docker-library/go-dockerlibrary/pkg/stripper"
  11. "pault.ag/go/debian/control"
  12. )
  13. var (
  14. GitCommitRegex = regexp.MustCompile(`^[0-9a-f]{1,40}$`)
  15. GitFetchRegex = regexp.MustCompile(`^refs/(heads|tags)/[^*?:]+$`)
  16. )
  17. type Manifest2822 struct {
  18. Global Manifest2822Entry
  19. Entries []Manifest2822Entry
  20. }
  21. type Manifest2822Entry struct {
  22. control.Paragraph
  23. Maintainers []string `delim:"," strip:"\n\r\t "`
  24. Tags []string `delim:"," strip:"\n\r\t "`
  25. SharedTags []string `delim:"," strip:"\n\r\t "`
  26. Architectures []string `delim:"," strip:"\n\r\t "`
  27. GitRepo string
  28. GitFetch string
  29. GitCommit string
  30. Directory string
  31. File string
  32. // architecture-specific versions of the above fields
  33. ArchValues map[string]string
  34. // "ARCH-FIELD: VALUE"
  35. // ala, "s390x-GitCommit: deadbeef"
  36. // (sourced from Paragraph.Values via .SeedArchValues())
  37. Constraints []string `delim:"," strip:"\n\r\t "`
  38. }
  39. var (
  40. DefaultArchitecture = "amd64"
  41. DefaultManifestEntry = Manifest2822Entry{
  42. Architectures: []string{DefaultArchitecture},
  43. GitFetch: "refs/heads/master",
  44. Directory: ".",
  45. File: "Dockerfile",
  46. }
  47. )
  48. func deepCopyStringsMap(a map[string]string) map[string]string {
  49. b := map[string]string{}
  50. for k, v := range a {
  51. b[k] = v
  52. }
  53. return b
  54. }
  55. func (entry Manifest2822Entry) Clone() Manifest2822Entry {
  56. // SLICES! grr
  57. entry.Maintainers = append([]string{}, entry.Maintainers...)
  58. entry.Tags = append([]string{}, entry.Tags...)
  59. entry.SharedTags = append([]string{}, entry.SharedTags...)
  60. entry.Architectures = append([]string{}, entry.Architectures...)
  61. entry.Constraints = append([]string{}, entry.Constraints...)
  62. // and MAPS, oh my
  63. entry.ArchValues = deepCopyStringsMap(entry.ArchValues)
  64. return entry
  65. }
  66. func (entry *Manifest2822Entry) SeedArchValues() {
  67. for field, val := range entry.Paragraph.Values {
  68. if strings.HasSuffix(field, "-GitRepo") || strings.HasSuffix(field, "-GitFetch") || strings.HasSuffix(field, "-GitCommit") || strings.HasSuffix(field, "-Directory") || strings.HasSuffix(field, "-File") {
  69. entry.ArchValues[field] = val
  70. }
  71. }
  72. }
  73. const StringSeparator2822 = ", "
  74. func (entry Manifest2822Entry) MaintainersString() string {
  75. return strings.Join(entry.Maintainers, StringSeparator2822)
  76. }
  77. func (entry Manifest2822Entry) TagsString() string {
  78. return strings.Join(entry.Tags, StringSeparator2822)
  79. }
  80. func (entry Manifest2822Entry) SharedTagsString() string {
  81. return strings.Join(entry.SharedTags, StringSeparator2822)
  82. }
  83. func (entry Manifest2822Entry) ArchitecturesString() string {
  84. return strings.Join(entry.Architectures, StringSeparator2822)
  85. }
  86. func (entry Manifest2822Entry) ConstraintsString() string {
  87. return strings.Join(entry.Constraints, StringSeparator2822)
  88. }
  89. // if this method returns "true", then a.Tags and b.Tags can safely be combined (for the purposes of building)
  90. func (a Manifest2822Entry) SameBuildArtifacts(b Manifest2822Entry) bool {
  91. // check xxxarch-GitRepo, etc. fields for sameness first
  92. for _, key := range append(a.archFields(), b.archFields()...) {
  93. if a.ArchValues[key] != b.ArchValues[key] {
  94. return false
  95. }
  96. }
  97. return a.ArchitecturesString() == b.ArchitecturesString() && a.GitRepo == b.GitRepo && a.GitFetch == b.GitFetch && a.GitCommit == b.GitCommit && a.Directory == b.Directory && a.File == b.File && a.ConstraintsString() == b.ConstraintsString()
  98. }
  99. // returns a list of architecture-specific fields in an Entry
  100. func (entry Manifest2822Entry) archFields() []string {
  101. ret := []string{}
  102. for key, val := range entry.ArchValues {
  103. if val != "" {
  104. ret = append(ret, key)
  105. }
  106. }
  107. sort.Strings(ret)
  108. return ret
  109. }
  110. // returns a new Entry with any of the values that are equal to the values in "defaults" cleared
  111. func (entry Manifest2822Entry) ClearDefaults(defaults Manifest2822Entry) Manifest2822Entry {
  112. entry = entry.Clone() // make absolutely certain we have a deep clone
  113. if entry.MaintainersString() == defaults.MaintainersString() {
  114. entry.Maintainers = nil
  115. }
  116. if entry.TagsString() == defaults.TagsString() {
  117. entry.Tags = nil
  118. }
  119. if entry.SharedTagsString() == defaults.SharedTagsString() {
  120. entry.SharedTags = nil
  121. }
  122. if entry.ArchitecturesString() == defaults.ArchitecturesString() {
  123. entry.Architectures = nil
  124. }
  125. if entry.GitRepo == defaults.GitRepo {
  126. entry.GitRepo = ""
  127. }
  128. if entry.GitFetch == defaults.GitFetch {
  129. entry.GitFetch = ""
  130. }
  131. if entry.GitCommit == defaults.GitCommit {
  132. entry.GitCommit = ""
  133. }
  134. if entry.Directory == defaults.Directory {
  135. entry.Directory = ""
  136. }
  137. if entry.File == defaults.File {
  138. entry.File = ""
  139. }
  140. for _, key := range defaults.archFields() {
  141. if defaults.ArchValues[key] == entry.ArchValues[key] {
  142. delete(entry.ArchValues, key)
  143. }
  144. }
  145. if entry.ConstraintsString() == defaults.ConstraintsString() {
  146. entry.Constraints = nil
  147. }
  148. return entry
  149. }
  150. func (entry Manifest2822Entry) String() string {
  151. ret := []string{}
  152. if str := entry.MaintainersString(); str != "" {
  153. ret = append(ret, "Maintainers: "+str)
  154. }
  155. if str := entry.TagsString(); str != "" {
  156. ret = append(ret, "Tags: "+str)
  157. }
  158. if str := entry.SharedTagsString(); str != "" {
  159. ret = append(ret, "SharedTags: "+str)
  160. }
  161. if str := entry.ArchitecturesString(); str != "" {
  162. ret = append(ret, "Architectures: "+str)
  163. }
  164. if str := entry.GitRepo; str != "" {
  165. ret = append(ret, "GitRepo: "+str)
  166. }
  167. if str := entry.GitFetch; str != "" {
  168. ret = append(ret, "GitFetch: "+str)
  169. }
  170. if str := entry.GitCommit; str != "" {
  171. ret = append(ret, "GitCommit: "+str)
  172. }
  173. if str := entry.Directory; str != "" {
  174. ret = append(ret, "Directory: "+str)
  175. }
  176. if str := entry.File; str != "" {
  177. ret = append(ret, "File: "+str)
  178. }
  179. for _, key := range entry.archFields() {
  180. ret = append(ret, key+": "+entry.ArchValues[key])
  181. }
  182. if str := entry.ConstraintsString(); str != "" {
  183. ret = append(ret, "Constraints: "+str)
  184. }
  185. return strings.Join(ret, "\n")
  186. }
  187. func (manifest Manifest2822) String() string {
  188. entries := []Manifest2822Entry{manifest.Global.ClearDefaults(DefaultManifestEntry)}
  189. entries = append(entries, manifest.Entries...)
  190. ret := []string{}
  191. for i, entry := range entries {
  192. if i > 0 {
  193. entry = entry.ClearDefaults(manifest.Global)
  194. }
  195. ret = append(ret, entry.String())
  196. }
  197. return strings.Join(ret, "\n\n")
  198. }
  199. func (entry *Manifest2822Entry) SetGitRepo(arch string, repo string) {
  200. if entry.ArchValues == nil {
  201. entry.ArchValues = map[string]string{}
  202. }
  203. entry.ArchValues[arch+"-GitRepo"] = repo
  204. }
  205. func (entry Manifest2822Entry) ArchGitRepo(arch string) string {
  206. if val, ok := entry.ArchValues[arch+"-GitRepo"]; ok && val != "" {
  207. return val
  208. }
  209. return entry.GitRepo
  210. }
  211. func (entry Manifest2822Entry) ArchGitFetch(arch string) string {
  212. if val, ok := entry.ArchValues[arch+"-GitFetch"]; ok && val != "" {
  213. return val
  214. }
  215. return entry.GitFetch
  216. }
  217. func (entry *Manifest2822Entry) SetGitCommit(arch string, commit string) {
  218. if entry.ArchValues == nil {
  219. entry.ArchValues = map[string]string{}
  220. }
  221. entry.ArchValues[arch+"-GitCommit"] = commit
  222. }
  223. func (entry Manifest2822Entry) ArchGitCommit(arch string) string {
  224. if val, ok := entry.ArchValues[arch+"-GitCommit"]; ok && val != "" {
  225. return val
  226. }
  227. return entry.GitCommit
  228. }
  229. func (entry Manifest2822Entry) ArchDirectory(arch string) string {
  230. if val, ok := entry.ArchValues[arch+"-Directory"]; ok && val != "" {
  231. return val
  232. }
  233. return entry.Directory
  234. }
  235. func (entry Manifest2822Entry) ArchFile(arch string) string {
  236. if val, ok := entry.ArchValues[arch+"-File"]; ok && val != "" {
  237. return val
  238. }
  239. return entry.File
  240. }
  241. func (entry Manifest2822Entry) HasTag(tag string) bool {
  242. for _, existingTag := range entry.Tags {
  243. if tag == existingTag {
  244. return true
  245. }
  246. }
  247. return false
  248. }
  249. // HasSharedTag returns true if the given tag exists in entry.SharedTags.
  250. func (entry Manifest2822Entry) HasSharedTag(tag string) bool {
  251. for _, existingTag := range entry.SharedTags {
  252. if tag == existingTag {
  253. return true
  254. }
  255. }
  256. return false
  257. }
  258. // HasArchitecture returns true if the given architecture exists in entry.Architectures
  259. func (entry Manifest2822Entry) HasArchitecture(arch string) bool {
  260. for _, existingArch := range entry.Architectures {
  261. if arch == existingArch {
  262. return true
  263. }
  264. }
  265. return false
  266. }
  267. func (manifest Manifest2822) GetTag(tag string) *Manifest2822Entry {
  268. for i, entry := range manifest.Entries {
  269. if entry.HasTag(tag) {
  270. return &manifest.Entries[i]
  271. }
  272. }
  273. return nil
  274. }
  275. // GetSharedTag returns a list of entries with the given tag in entry.SharedTags (or the empty list if there are no entries with the given tag).
  276. func (manifest Manifest2822) GetSharedTag(tag string) []*Manifest2822Entry {
  277. ret := []*Manifest2822Entry{}
  278. for i, entry := range manifest.Entries {
  279. if entry.HasSharedTag(tag) {
  280. ret = append(ret, &manifest.Entries[i])
  281. }
  282. }
  283. return ret
  284. }
  285. // GetAllSharedTags returns a list of the sum of all SharedTags in all entries of this image manifest (in the order they appear in the file).
  286. func (manifest Manifest2822) GetAllSharedTags() []string {
  287. fakeEntry := Manifest2822Entry{}
  288. for _, entry := range manifest.Entries {
  289. fakeEntry.SharedTags = append(fakeEntry.SharedTags, entry.SharedTags...)
  290. }
  291. fakeEntry.DeduplicateSharedTags()
  292. return fakeEntry.SharedTags
  293. }
  294. type SharedTagGroup struct {
  295. SharedTags []string
  296. Entries []*Manifest2822Entry
  297. }
  298. // GetSharedTagGroups returns a map of shared tag groups to the list of entries they share (as described in https://github.com/docker-library/go-dockerlibrary/pull/2#issuecomment-277853597).
  299. func (manifest Manifest2822) GetSharedTagGroups() []SharedTagGroup {
  300. inter := map[string][]string{}
  301. interOrder := []string{} // order matters, and maps randomize order
  302. interKeySep := ","
  303. for _, sharedTag := range manifest.GetAllSharedTags() {
  304. interKeyParts := []string{}
  305. for _, entry := range manifest.GetSharedTag(sharedTag) {
  306. interKeyParts = append(interKeyParts, entry.Tags[0])
  307. }
  308. interKey := strings.Join(interKeyParts, interKeySep)
  309. if _, ok := inter[interKey]; !ok {
  310. interOrder = append(interOrder, interKey)
  311. }
  312. inter[interKey] = append(inter[interKey], sharedTag)
  313. }
  314. ret := []SharedTagGroup{}
  315. for _, tags := range interOrder {
  316. group := SharedTagGroup{
  317. SharedTags: inter[tags],
  318. Entries: []*Manifest2822Entry{},
  319. }
  320. for _, tag := range strings.Split(tags, interKeySep) {
  321. group.Entries = append(group.Entries, manifest.GetTag(tag))
  322. }
  323. ret = append(ret, group)
  324. }
  325. return ret
  326. }
  327. func (manifest *Manifest2822) AddEntry(entry Manifest2822Entry) error {
  328. if len(entry.Tags) < 1 {
  329. return fmt.Errorf("missing Tags")
  330. }
  331. if entry.GitRepo == "" || entry.GitFetch == "" || entry.GitCommit == "" {
  332. return fmt.Errorf("Tags %q missing one of GitRepo, GitFetch, or GitCommit", entry.TagsString())
  333. }
  334. if invalidMaintainers := entry.InvalidMaintainers(); len(invalidMaintainers) > 0 {
  335. return fmt.Errorf("Tags %q has invalid Maintainers: %q (expected format %q)", entry.TagsString(), strings.Join(invalidMaintainers, ", "), MaintainersFormat)
  336. }
  337. entry.DeduplicateSharedTags()
  338. if invalidArchitectures := entry.InvalidArchitectures(); len(invalidArchitectures) > 0 {
  339. return fmt.Errorf("Tags %q has invalid Architectures: %q", entry.TagsString(), strings.Join(invalidArchitectures, ", "))
  340. }
  341. seenTag := map[string]bool{}
  342. for _, tag := range entry.Tags {
  343. if otherEntry := manifest.GetTag(tag); otherEntry != nil {
  344. return fmt.Errorf("Tags %q includes duplicate tag: %q (duplicated in %q)", entry.TagsString(), tag, otherEntry.TagsString())
  345. }
  346. if otherEntries := manifest.GetSharedTag(tag); len(otherEntries) > 0 {
  347. return fmt.Errorf("Tags %q includes tag conflicting with a shared tag: %q (shared tag in %q)", entry.TagsString(), tag, otherEntries[0].TagsString())
  348. }
  349. if seenTag[tag] {
  350. return fmt.Errorf("Tags %q includes duplicate tag: %q", entry.TagsString(), tag)
  351. }
  352. seenTag[tag] = true
  353. }
  354. for _, tag := range entry.SharedTags {
  355. if otherEntry := manifest.GetTag(tag); otherEntry != nil {
  356. return fmt.Errorf("Tags %q includes conflicting shared tag: %q (duplicated in %q)", entry.TagsString(), tag, otherEntry.TagsString())
  357. }
  358. if seenTag[tag] {
  359. return fmt.Errorf("Tags %q includes duplicate tag: %q (in SharedTags)", entry.TagsString(), tag)
  360. }
  361. seenTag[tag] = true
  362. }
  363. for i, existingEntry := range manifest.Entries {
  364. if existingEntry.SameBuildArtifacts(entry) {
  365. manifest.Entries[i].Tags = append(existingEntry.Tags, entry.Tags...)
  366. manifest.Entries[i].SharedTags = append(existingEntry.SharedTags, entry.SharedTags...)
  367. manifest.Entries[i].DeduplicateSharedTags()
  368. return nil
  369. }
  370. }
  371. manifest.Entries = append(manifest.Entries, entry)
  372. return nil
  373. }
  374. const (
  375. MaintainersNameRegex = `[^\s<>()][^<>()]*`
  376. MaintainersEmailRegex = `[^\s<>()]+`
  377. MaintainersGitHubRegex = `[^\s<>()]+`
  378. MaintainersFormat = `Full Name <contact-email-or-url> (@github-handle) OR Full Name (@github-handle)`
  379. )
  380. var (
  381. MaintainersRegex = regexp.MustCompile(`^(` + MaintainersNameRegex + `)(?:\s+<(` + MaintainersEmailRegex + `)>)?\s+[(]@(` + MaintainersGitHubRegex + `)[)]$`)
  382. )
  383. func (entry Manifest2822Entry) InvalidMaintainers() []string {
  384. invalid := []string{}
  385. for _, maintainer := range entry.Maintainers {
  386. if !MaintainersRegex.MatchString(maintainer) {
  387. invalid = append(invalid, maintainer)
  388. }
  389. }
  390. return invalid
  391. }
  392. func (entry Manifest2822Entry) InvalidArchitectures() []string {
  393. invalid := []string{}
  394. for _, arch := range entry.Architectures {
  395. if _, ok := architecture.SupportedArches[arch]; !ok {
  396. invalid = append(invalid, arch)
  397. }
  398. }
  399. return invalid
  400. }
  401. // DeduplicateSharedTags will remove duplicate values from entry.SharedTags, preserving order.
  402. func (entry *Manifest2822Entry) DeduplicateSharedTags() {
  403. aggregate := []string{}
  404. seen := map[string]bool{}
  405. for _, tag := range entry.SharedTags {
  406. if seen[tag] {
  407. continue
  408. }
  409. seen[tag] = true
  410. aggregate = append(aggregate, tag)
  411. }
  412. entry.SharedTags = aggregate
  413. }
  414. // DeduplicateArchitectures will remove duplicate values from entry.Architectures and sort the result.
  415. func (entry *Manifest2822Entry) DeduplicateArchitectures() {
  416. aggregate := []string{}
  417. seen := map[string]bool{}
  418. for _, arch := range entry.Architectures {
  419. if seen[arch] {
  420. continue
  421. }
  422. seen[arch] = true
  423. aggregate = append(aggregate, arch)
  424. }
  425. sort.Strings(aggregate)
  426. entry.Architectures = aggregate
  427. }
  428. type decoderWrapper struct {
  429. *control.Decoder
  430. }
  431. func (decoder *decoderWrapper) Decode(entry *Manifest2822Entry) error {
  432. // reset Architectures and SharedTags so that they can be either inherited or replaced, not additive
  433. sharedTags := entry.SharedTags
  434. entry.SharedTags = nil
  435. arches := entry.Architectures
  436. entry.Architectures = nil
  437. for {
  438. err := decoder.Decoder.Decode(entry)
  439. if err != nil {
  440. return err
  441. }
  442. // ignore empty paragraphs (blank lines at the start, excess blank lines between paragraphs, excess blank lines at EOF)
  443. if len(entry.Paragraph.Order) == 0 {
  444. continue
  445. }
  446. // if we had no SharedTags or Architectures, restore our "default" (original) values
  447. if len(entry.SharedTags) == 0 {
  448. entry.SharedTags = sharedTags
  449. }
  450. if len(entry.Architectures) == 0 {
  451. entry.Architectures = arches
  452. }
  453. entry.DeduplicateArchitectures()
  454. // pull out any new architecture-specific values from Paragraph.Values
  455. entry.SeedArchValues()
  456. return nil
  457. }
  458. }
  459. func Parse2822(readerIn io.Reader) (*Manifest2822, error) {
  460. reader := stripper.NewCommentStripper(readerIn)
  461. realDecoder, err := control.NewDecoder(bufio.NewReader(reader), nil)
  462. if err != nil {
  463. return nil, err
  464. }
  465. decoder := decoderWrapper{realDecoder}
  466. manifest := Manifest2822{
  467. Global: DefaultManifestEntry.Clone(),
  468. }
  469. if err := decoder.Decode(&manifest.Global); err != nil {
  470. return nil, err
  471. }
  472. if len(manifest.Global.Maintainers) < 1 {
  473. return nil, fmt.Errorf("missing Maintainers")
  474. }
  475. if invalidMaintainers := manifest.Global.InvalidMaintainers(); len(invalidMaintainers) > 0 {
  476. return nil, fmt.Errorf("invalid Maintainers: %q (expected format %q)", strings.Join(invalidMaintainers, ", "), MaintainersFormat)
  477. }
  478. if len(manifest.Global.Tags) > 0 {
  479. return nil, fmt.Errorf("global Tags not permitted")
  480. }
  481. if invalidArchitectures := manifest.Global.InvalidArchitectures(); len(invalidArchitectures) > 0 {
  482. return nil, fmt.Errorf("invalid global Architectures: %q", strings.Join(invalidArchitectures, ", "))
  483. }
  484. for {
  485. entry := manifest.Global.Clone()
  486. err := decoder.Decode(&entry)
  487. if err == io.EOF {
  488. break
  489. }
  490. if err != nil {
  491. return nil, err
  492. }
  493. if !GitFetchRegex.MatchString(entry.GitFetch) {
  494. return nil, fmt.Errorf(`Tags %q has invalid GitFetch (must be "refs/heads/..." or "refs/tags/..."): %q`, entry.TagsString(), entry.GitFetch)
  495. }
  496. if !GitCommitRegex.MatchString(entry.GitCommit) {
  497. return nil, fmt.Errorf(`Tags %q has invalid GitCommit (must be a commit, not a tag or ref): %q`, entry.TagsString(), entry.GitCommit)
  498. }
  499. err = manifest.AddEntry(entry)
  500. if err != nil {
  501. return nil, err
  502. }
  503. }
  504. return &manifest, nil
  505. }