feeds 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649
  1. #!/usr/bin/perl
  2. use Getopt::Std;
  3. use FindBin;
  4. use Cwd;
  5. use lib "$FindBin::Bin";
  6. use metadata;
  7. use warnings;
  8. use strict;
  9. use Cwd 'abs_path';
  10. chdir "$FindBin::Bin/..";
  11. $ENV{TOPDIR}=getcwd();
  12. $ENV{GIT_CONFIG_PARAMETERS}="'core.autocrlf=false'";
  13. my $mk=`which gmake 2>/dev/null`; # select the right 'make' program
  14. chomp($mk); # trim trailing newline
  15. $mk or $mk = "make"; # default to 'make'
  16. # check version of make
  17. my @mkver = split /\s+/, `$mk -v`, 4;
  18. my $valid_mk = 1;
  19. $mkver[0] =~ /^GNU/ or $valid_mk = 0;
  20. $mkver[1] =~ /^Make/ or $valid_mk = 0;
  21. $mkver[2] >= "3.81" or $valid_mk = 0;
  22. $valid_mk or die "Unsupported version of make found: $mk\n";
  23. my @feeds;
  24. my %build_packages;
  25. my %installed;
  26. my %feed_cache;
  27. my $feed_package = {};
  28. my $feed_src = {};
  29. sub parse_config() {
  30. my $line = 0;
  31. my %name;
  32. open FEEDS, "feeds.conf" or
  33. open FEEDS, "feeds.conf.default" or
  34. die "Unable to open feeds configuration";
  35. while (<FEEDS>) {
  36. chomp;
  37. s/#.+$//;
  38. next unless /\S/;
  39. my @line = split /\s+/, $_, 3;
  40. my @src;
  41. $line++;
  42. my $valid = 1;
  43. $line[0] =~ /^src-\w+$/ or $valid = 0;
  44. $line[1] =~ /^\w+$/ or $valid = 0;
  45. @src = split /\s+/, $line[2];
  46. $valid or die "Syntax error in feeds.conf, line: $line\n";
  47. $name{$line[1]} and die "Duplicate feed name '$line[1]', line: $line\n";
  48. $name{$line[1]} = 1;
  49. push @feeds, [$line[0], $line[1], \@src];
  50. }
  51. close FEEDS;
  52. }
  53. sub update_location($$)
  54. {
  55. my $name = shift;
  56. my $url = shift;
  57. my $old_url;
  58. -d "./feeds/$name.tmp" or mkdir "./feeds/$name.tmp" or return 1;
  59. if( open LOC, "< ./feeds/$name.tmp/location" )
  60. {
  61. chomp($old_url = readline LOC);
  62. close LOC;
  63. }
  64. if( !$old_url || $old_url ne $url )
  65. {
  66. if( open LOC, "> ./feeds/$name.tmp/location" )
  67. {
  68. print LOC $url, "\n";
  69. close LOC;
  70. }
  71. return $old_url ? 1 : 0;
  72. }
  73. return 0;
  74. }
  75. sub update_index($)
  76. {
  77. my $name = shift;
  78. -d "./feeds/$name.tmp" or mkdir "./feeds/$name.tmp" or return 1;
  79. -d "./feeds/$name.tmp/info" or mkdir "./feeds/$name.tmp/info" or return 1;
  80. system("$mk -s prepare-mk OPENWRT_BUILD= TMP_DIR=\"$ENV{TOPDIR}/feeds/$name.tmp\"");
  81. system("$mk -s -f include/scan.mk IS_TTY=1 SCAN_TARGET=\"packageinfo\" SCAN_DIR=\"feeds/$name\" SCAN_NAME=\"package\" SCAN_DEPS=\"$ENV{TOPDIR}/include/package*.mk\" SCAN_DEPTH=5 SCAN_EXTRA=\"\" TMP_DIR=\"$ENV{TOPDIR}/feeds/$name.tmp\"");
  82. system("ln -sf $name.tmp/.packageinfo ./feeds/$name.index");
  83. return 0;
  84. }
  85. my %update_method = (
  86. 'src-svn' => {
  87. 'init' => "svn checkout '%s' '%s'",
  88. 'update' => "svn update",
  89. 'controldir' => ".svn",
  90. 'revision' => "svn info | grep 'Revision' | cut -d ' ' -f 2 | tr -d '\n'"},
  91. 'src-cpy' => {
  92. 'init' => "cp -Rf '%s' '%s'",
  93. 'update' => "",
  94. 'revision' => "echo -n 'local'"},
  95. 'src-link' => {
  96. 'init' => "ln -s '%s' '%s'",
  97. 'update' => "",
  98. 'revision' => "echo -n 'local'"},
  99. 'src-git' => {
  100. 'init' => "git clone --depth 1 '%s' '%s'",
  101. 'init_branch' => "git clone --depth 1 --branch '%s' '%s' '%s'",
  102. 'update' => "git pull",
  103. 'controldir' => ".git",
  104. 'revision' => "git show --abbrev-commit HEAD | head -n 1 | cut -d ' ' -f 2 | tr -d '\n'"},
  105. 'src-gitsvn' => {
  106. 'init' => "git svn clone -r HEAD '%s' '%s'",
  107. 'update' => "git svn rebase",
  108. 'controldir' => ".git",
  109. 'revision' => "git show --abbrev-commit HEAD | head -n 1 | cut -d ' ' -f 2 | tr -d '\n'"},
  110. 'src-bzr' => {
  111. 'init' => "bzr checkout --lightweight '%s' '%s'",
  112. 'update' => "bzr update",
  113. 'controldir' => ".bzr"},
  114. 'src-hg' => {
  115. 'init' => "hg clone '%s' '%s'",
  116. 'update' => "hg pull --update",
  117. 'controldir' => ".hg"},
  118. 'src-darcs' => {
  119. 'init' => "darcs get '%s' '%s'",
  120. 'update' => "darcs pull -a",
  121. 'controldir' => "_darcs"},
  122. );
  123. # src-git: pull broken
  124. # src-cpy: broken if `basename $src` != $name
  125. sub update_feed_via($$$$) {
  126. my $type = shift;
  127. my $name = shift;
  128. my $src = shift;
  129. my $relocate = shift;
  130. my $m = $update_method{$type};
  131. my $localpath = "./feeds/$name";
  132. my $safepath = $localpath;
  133. $safepath =~ s/'/'\\''/;
  134. my ($base, $branch) = split(/;/, $src, 2);
  135. if( $relocate || !$m->{'update'} || !-d "$localpath/$m->{'controldir'}" ) {
  136. system("rm -rf '$safepath'");
  137. if ($m->{'init_branch'} and $branch) {
  138. system(sprintf($m->{'init_branch'}, $branch, $base, $safepath)) == 0 or return 1;
  139. } else {
  140. system(sprintf($m->{'init'}, $src, $safepath)) == 0 or return 1;
  141. }
  142. } else {
  143. system("cd '$safepath'; $m->{'update'}") == 0 or return 1;
  144. }
  145. return 0;
  146. }
  147. sub get_feed($) {
  148. my $feed = shift;
  149. if (!defined($feed_cache{$feed})) {
  150. my $file = "./feeds/$feed.index";
  151. clear_packages();
  152. -f $file or do {
  153. print "Ignoring feed '$feed' - index missing\n";
  154. return;
  155. };
  156. parse_package_metadata($file) or return;
  157. $feed_cache{$feed} = [ { %package }, { %srcpackage } ];
  158. }
  159. $feed_package = $feed_cache{$feed}->[0];
  160. $feed_src = $feed_cache{$feed}->[1];
  161. return $feed_cache{$feed}->[0];
  162. }
  163. sub get_installed() {
  164. system("$mk -s prepare-tmpinfo OPENWRT_BUILD=");
  165. clear_packages();
  166. parse_package_metadata("./tmp/.packageinfo");
  167. %installed = %package;
  168. }
  169. sub search_feed {
  170. my $feed = shift;
  171. my @substr = @_;
  172. my $display;
  173. return unless @substr > 0;
  174. get_feed($feed);
  175. foreach my $name (sort { lc($a) cmp lc($b) } keys %$feed_package) {
  176. my $pkg = $feed_package->{$name};
  177. my $substr;
  178. my $pkgmatch = 1;
  179. next if $pkg->{vdepends};
  180. foreach my $substr (@substr) {
  181. my $match;
  182. foreach my $key (qw(name title description src)) {
  183. $pkg->{$key} and $substr and $pkg->{$key} =~ m/$substr/i and $match = 1;
  184. }
  185. $match or undef $pkgmatch;
  186. };
  187. $pkgmatch and do {
  188. $display or do {
  189. print "Search results in feed '$feed':\n";
  190. $display = 1;
  191. };
  192. printf "\%-25s\t\%s\n", $pkg->{name}, $pkg->{title};
  193. };
  194. }
  195. return 0;
  196. }
  197. sub search {
  198. my %opts;
  199. getopt('r:', \%opts);
  200. foreach my $feed (@feeds) {
  201. search_feed($feed->[1], @ARGV) if (!defined($opts{r}) or $opts{r} eq $feed->[1]);
  202. }
  203. }
  204. sub list_feed {
  205. my $feed = shift;
  206. get_feed($feed);
  207. foreach my $name (sort { lc($a) cmp lc($b) } keys %$feed_package) {
  208. my $pkg = $feed_package->{$name};
  209. next if $pkg->{vdepends};
  210. if($pkg->{name}) {
  211. printf "\%-32s\t\%s\n", $pkg->{name}, $pkg->{title};
  212. }
  213. }
  214. return 0;
  215. }
  216. sub list {
  217. my %opts;
  218. getopts('r:d:sh', \%opts);
  219. if ($opts{h}) {
  220. usage();
  221. return 0;
  222. }
  223. if ($opts{s}) {
  224. foreach my $feed (@feeds) {
  225. my $localpath = "./feeds/$feed->[1]";
  226. my $m = $update_method{$feed->[0]};
  227. my $revision;
  228. if( !$m->{'revision'} ) {
  229. $revision = "X";
  230. }
  231. elsif( $m->{'controldir'} && -d "$localpath/$m->{'controldir'}" ) {
  232. $revision = `cd '$localpath'; $m->{'revision'}`;
  233. }
  234. else {
  235. $revision = "local";
  236. }
  237. if ($opts{d}) {
  238. printf "%s%s%s%s%s%s%s\n", $feed->[1], $opts{d}, $feed->[0], $opts{d}, $revision, $opts{d}, join(", ", @{$feed->[2]});
  239. }
  240. else {
  241. printf "\%-8s \%-8s \%-8s \%s\n", $feed->[1], $feed->[0], $revision, join(", ", @{$feed->[2]});
  242. }
  243. }
  244. return 0;
  245. }
  246. foreach my $feed (@feeds) {
  247. list_feed($feed->[1], @ARGV) if (!defined($opts{r}) or $opts{r} eq $feed->[1]);
  248. }
  249. return 0;
  250. }
  251. sub install_generic() {
  252. my $feed = shift;
  253. my $pkg = shift;
  254. my $path = $pkg->{makefile};
  255. if($path) {
  256. $path =~ s/\/Makefile$//;
  257. -d "./package/feeds" or mkdir "./package/feeds";
  258. -d "./package/feeds/$feed->[1]" or mkdir "./package/feeds/$feed->[1]";
  259. system("ln -sf ../../../$path ./package/feeds/$feed->[1]/");
  260. } else {
  261. warn "Package is not valid\n";
  262. return 1;
  263. }
  264. return 0;
  265. }
  266. my %install_method = (
  267. 'src-svn' => \&install_generic,
  268. 'src-cpy' => \&install_generic,
  269. 'src-link' => \&install_generic,
  270. 'src-git' => \&install_generic,
  271. 'src-gitsvn' => \&install_generic,
  272. 'src-bzr' => \&install_generic,
  273. 'src-hg' => \&install_generic,
  274. 'src-darcs' => \&install_generic,
  275. );
  276. my %feed;
  277. sub lookup_package($$) {
  278. my $feed = shift;
  279. my $package = shift;
  280. foreach my $feed ($feed, @feeds) {
  281. next unless $feed->[1];
  282. next unless $feed{$feed->[1]};
  283. $feed{$feed->[1]}->{$package} and return $feed;
  284. }
  285. return;
  286. }
  287. sub is_core_package($) {
  288. my $package = shift;
  289. foreach my $file ("tmp/info/.packageinfo-$package", glob("tmp/info/.packageinfo-*_$package")) {
  290. next unless index($file, "tmp/info/.packageinfo-feeds_");
  291. return 1 if -s $file;
  292. }
  293. return 0;
  294. }
  295. sub install_package {
  296. my $feed = shift;
  297. my $name = shift;
  298. my $ret = 0;
  299. $feed = lookup_package($feed, $name);
  300. $feed or do {
  301. $installed{$name} and return 0;
  302. # TODO: check if it's already installed within ./package directory
  303. $feed_src->{$name} or is_core_package($name) or warn "WARNING: No feed for package '$name' found, maybe it's already part of the standard packages?\n";
  304. return 0;
  305. };
  306. # switch to the metadata for the selected feed
  307. get_feed($feed->[1]);
  308. my $pkg = $feed{$feed->[1]}->{$name} or return 1;
  309. $pkg->{name} or do {
  310. $installed{$name} and return 0;
  311. # TODO: check if this is an alias package, maybe it's known by another name
  312. warn "WARNING: Package '$name' is not available in feed $feed->[1].\n";
  313. return 0;
  314. };
  315. my $src = $pkg->{src};
  316. my $type = $feed->[0];
  317. $src or $src = $name;
  318. # previously installed packages set the runtime package
  319. # newly installed packages set the source package
  320. $installed{$src} and return 0;
  321. # check previously installed packages
  322. $installed{$name} and return 0;
  323. $installed{$src} = 1;
  324. warn "Installing package '$src'\n";
  325. $install_method{$type} or do {
  326. warn "Unknown installation method: '$type'\n";
  327. return 1;
  328. };
  329. &{$install_method{$type}}($feed, $pkg) == 0 or do {
  330. warn "failed.\n";
  331. return 1;
  332. };
  333. # install all dependencies referenced from the source package
  334. foreach my $vpkg (@{$feed_src->{$src}}) {
  335. foreach my $dep (@{$vpkg->{depends}}, @{$vpkg->{builddepends}}, @{$vpkg->{"builddepends/host"}}) {
  336. next if $dep =~ /@/;
  337. $dep =~ s/^\+//;
  338. $dep =~ s/^.+://;
  339. $dep =~ s/\/.+$//;
  340. next unless $dep;
  341. install_package($feed, $dep) == 0 or $ret = 1;
  342. }
  343. }
  344. return $ret;
  345. }
  346. sub refresh_config {
  347. my $default = shift;
  348. # workaround for timestamp check
  349. system("rm -f tmp/.packageinfo");
  350. # refresh the config
  351. if ($default) {
  352. system("$mk oldconfig CONFDEFAULT=\"$default\" Config.in >/dev/null 2>/dev/null");
  353. } else {
  354. system("$mk defconfig Config.in >/dev/null 2>/dev/null");
  355. }
  356. }
  357. sub install {
  358. my $name;
  359. my %opts;
  360. my $feed;
  361. my $ret = 0;
  362. getopts('ap:d:h', \%opts);
  363. if ($opts{h}) {
  364. usage();
  365. return 0;
  366. }
  367. get_installed();
  368. foreach my $f (@feeds) {
  369. # index all feeds
  370. $feed{$f->[1]} = get_feed($f->[1]);
  371. # look up the preferred feed
  372. $opts{p} and $f->[1] eq $opts{p} and $feed = $f;
  373. }
  374. if($opts{a}) {
  375. foreach my $f (@feeds) {
  376. if (!defined($opts{p}) or $opts{p} eq $f->[1]) {
  377. printf "Installing all packages from feed %s.\n", $f->[1];
  378. get_feed($f->[1]);
  379. foreach my $name (sort { lc($a) cmp lc($b) } keys %$feed_package) {
  380. my $p = $feed_package->{$name};
  381. next if $p->{vdepends};
  382. if( $p->{name} ) {
  383. install_package($feed, $p->{name}) == 0 or $ret = 1;
  384. get_feed($f->[1]);
  385. }
  386. }
  387. }
  388. }
  389. } else {
  390. while ($name = shift @ARGV) {
  391. install_package($feed, $name) == 0 or $ret = 1;
  392. }
  393. }
  394. # workaround for timestamp check
  395. # set the defaults
  396. if ($opts{d} and $opts{d} =~ /^[ymn]$/) {
  397. refresh_config($opts{d});
  398. }
  399. return $ret;
  400. }
  401. sub uninstall {
  402. my %opts;
  403. my $name;
  404. my $uninstall;
  405. getopts('ah', \%opts);
  406. if ($opts{h}) {
  407. usage();
  408. return 0;
  409. }
  410. if ($opts{a}) {
  411. system("rm -rvf ./package/feeds");
  412. $uninstall = 1;
  413. } else {
  414. if($#ARGV == -1) {
  415. warn "WARNING: no package to uninstall\n";
  416. return 0;
  417. }
  418. get_installed();
  419. while ($name = shift @ARGV) {
  420. my $pkg = $installed{$name};
  421. $pkg or do {
  422. warn "WARNING: $name not installed\n";
  423. next;
  424. };
  425. $pkg->{src} and $name = $pkg->{src};
  426. warn "Uninstalling package '$name'\n";
  427. system("rm -f ./package/feeds/*/$name");
  428. $uninstall = 1;
  429. }
  430. }
  431. $uninstall and refresh_config();
  432. return 0;
  433. }
  434. sub update_feed($$$$)
  435. {
  436. my $type=shift;
  437. my $name=shift;
  438. my $src=shift;
  439. my $perform_update=shift;
  440. my $force_relocate=update_location( $name, "@$src" );
  441. if( $force_relocate ) {
  442. warn "Source of feed $name has changed, replacing copy\n";
  443. }
  444. $update_method{$type} or do {
  445. warn "Unknown type '$type' in feed $name\n";
  446. return 1;
  447. };
  448. $perform_update and do {
  449. my $failed = 1;
  450. foreach my $feedsrc (@$src) {
  451. warn "Updating feed '$name' from '$feedsrc' ...\n";
  452. next unless update_feed_via($type, $name, $feedsrc, $force_relocate) == 0;
  453. $failed = 0;
  454. last;
  455. }
  456. $failed and do {
  457. warn "failed.\n";
  458. return 1;
  459. };
  460. };
  461. warn "Create index file './feeds/$name.index' \n";
  462. update_index($name) == 0 or do {
  463. warn "failed.\n";
  464. return 1;
  465. };
  466. return 0;
  467. }
  468. sub update {
  469. my %opts;
  470. my $feed_name;
  471. my $perform_update=1;
  472. $ENV{SCAN_COOKIE} = $$;
  473. $ENV{OPENWRT_VERBOSE} = 's';
  474. getopts('ahi', \%opts);
  475. if ($opts{h}) {
  476. usage();
  477. return 0;
  478. }
  479. if ($opts{i}) {
  480. # don't update from (remote) repository
  481. # only re-create index information
  482. $perform_update=0;
  483. }
  484. -d "feeds" or do {
  485. mkdir "feeds" or die "Unable to create the feeds directory";
  486. };
  487. if ( ($#ARGV == -1) or $opts{a}) {
  488. foreach my $feed (@feeds) {
  489. my ($type, $name, $src) = @$feed;
  490. update_feed($type, $name, $src, $perform_update);
  491. }
  492. } else {
  493. while ($feed_name = shift @ARGV) {
  494. foreach my $feed (@feeds) {
  495. my ($type, $name, $src) = @$feed;
  496. if($feed_name ne $name) {
  497. next;
  498. }
  499. update_feed($type, $name, $src, $perform_update);
  500. }
  501. }
  502. }
  503. refresh_config();
  504. return 0;
  505. }
  506. sub usage() {
  507. print <<EOF;
  508. Usage: $0 <command> [options]
  509. Commands:
  510. list [options]: List feeds, their content and revisions (if installed)
  511. Options:
  512. -s : List of feed names and their URL.
  513. -r <feedname>: List packages of specified feed.
  514. -d <delimiter>: Use specified delimiter to distinguish rows (default: spaces)
  515. install [options] <package>: Install a package
  516. Options:
  517. -a : Install all packages from all feeds or from the specified feed using the -p option.
  518. -p <feedname>: Prefer this feed when installing packages.
  519. -d <y|m|n>: Set default for newly installed packages.
  520. search [options] <substring>: Search for a package
  521. Options:
  522. -r <feedname>: Only search in this feed
  523. uninstall -a|<package>: Uninstall a package
  524. Options:
  525. -a : Uninstalls all packages.
  526. update -a|<feedname(s)>: Update packages and lists of feeds in feeds.conf .
  527. Options:
  528. -a : Update all feeds listed within feeds.conf. Otherwise the specified feeds will be updated.
  529. -i : Recreate the index only. No feed update from repository is performed.
  530. clean: Remove downloaded/generated files.
  531. EOF
  532. exit(1);
  533. }
  534. my %commands = (
  535. 'list' => \&list,
  536. 'update' => \&update,
  537. 'install' => \&install,
  538. 'search' => \&search,
  539. 'uninstall' => \&uninstall,
  540. 'clean' => sub {
  541. system("rm -rf feeds");
  542. }
  543. );
  544. my $arg = shift @ARGV;
  545. $arg or usage();
  546. parse_config;
  547. foreach my $cmd (keys %commands) {
  548. $arg eq $cmd and do {
  549. exit(&{$commands{$cmd}}());
  550. };
  551. }
  552. usage();