summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAdam Coldrick <adam.coldrick@codethink.co.uk>2015-04-15 12:17:16 (GMT)
committerMorph (on behalf of Adam Coldrick) <adam.coldrick@codethink.co.uk>2015-04-15 12:17:16 (GMT)
commit85995d210162d1432800acf357f8162b77f5b47e (patch)
tree256ba048782865a8d9b52e497ff0ea5f694d652b
parent3167ced4844c9602e88289607d1c2cc2ecbd5d95 (diff)
downloadmorph-baserock/6453f312359f4317803ef7f14b58d21f/d675b946df4f456693ed211dcd2ec95e.tar.gz
System branch: master
-rw-r--r--COPYING339
-rw-r--r--README189
-rw-r--r--distbuild/build_controller.py129
-rw-r--r--distbuild/initiator.py8
-rw-r--r--distbuild/json_router.py1
-rw-r--r--distbuild/protocol.py6
-rw-r--r--distbuild/worker_build_scheduler.py19
-rw-r--r--morphlib/__init__.py5
-rw-r--r--morphlib/app.py30
-rw-r--r--morphlib/buildbranch.py7
-rw-r--r--morphlib/buildcommand.py40
-rw-r--r--morphlib/buildenvironment.py22
-rw-r--r--morphlib/builder.py34
-rw-r--r--morphlib/cachedrepo.py20
-rwxr-xr-xmorphlib/exts/fstab.configure25
-rwxr-xr-xmorphlib/exts/hosts.configure48
-rwxr-xr-xmorphlib/exts/install-files.configure42
-rwxr-xr-xmorphlib/exts/kvm.check23
-rwxr-xr-xmorphlib/exts/simple-network.configure39
-rw-r--r--morphlib/gitdir.py26
-rw-r--r--morphlib/gitdir_tests.py32
-rw-r--r--morphlib/localartifactcache.py5
-rw-r--r--morphlib/ostreeartifactcache.py147
-rw-r--r--morphlib/plugins/branch_and_merge_plugin.py1
-rw-r--r--morphlib/plugins/build_plugin.py137
-rw-r--r--morphlib/plugins/certify_plugin.py140
-rw-r--r--morphlib/plugins/cross-bootstrap_plugin.py2
-rw-r--r--morphlib/plugins/deploy_plugin.py223
-rw-r--r--morphlib/plugins/gc_plugin.py3
-rw-r--r--morphlib/plugins/get_chunk_details_plugin.py79
-rw-r--r--morphlib/plugins/ostree_artifacts_plugin.py169
-rw-r--r--morphlib/sourceresolver.py84
-rw-r--r--morphlib/stagingarea.py58
-rw-r--r--morphlib/stagingarea_tests.py27
-rw-r--r--morphlib/util.py57
-rw-r--r--morphlib/writeexts.py6
-rwxr-xr-xscripts/check-copyright-year4
-rwxr-xr-xtests.build/build-chunk-writes-log.script35
-rwxr-xr-xtests.build/build-stratum-with-submodules.script15
-rw-r--r--tests.build/build-stratum-with-submodules.stdout2
-rwxr-xr-xtests.build/build-system-autotools.script13
-rw-r--r--tests.build/build-system-autotools.stdout3
-rwxr-xr-xtests.build/build-system-cmake.script14
-rw-r--r--tests.build/build-system-cmake.stdout2
-rwxr-xr-xtests.build/build-system-cpan.script14
-rw-r--r--tests.build/build-system-cpan.stdout1
-rwxr-xr-xtests.build/build-system-python-distutils.script19
-rw-r--r--tests.build/build-system-python-distutils.stdout6
-rwxr-xr-xtests.build/build-system-qmake.script8
-rw-r--r--tests.build/build-system-qmake.stdout8
-rwxr-xr-xtests.build/build-system.script35
-rw-r--r--tests.build/build-system.stdout2
-rwxr-xr-xtests.build/cross-bootstrap.script4
-rwxr-xr-xtests.build/morphless-chunks.script13
-rw-r--r--tests.build/morphless-chunks.stdout0
-rw-r--r--tests.build/only-build-systems.stderr4
-rwxr-xr-xtests.build/prefix.script14
-rw-r--r--tests.build/prefix.stdout8
-rwxr-xr-xtests.build/rebuild-cached-stratum.script7
-rw-r--r--tests.build/rebuild-cached-stratum.stdout10
-rw-r--r--without-test-modules3
-rw-r--r--yarns/building.yarn11
-rw-r--r--yarns/deployment.yarn22
-rw-r--r--yarns/implementations.yarn39
64 files changed, 2015 insertions, 523 deletions
diff --git a/COPYING b/COPYING
new file mode 100644
index 0000000..d159169
--- /dev/null
+++ b/COPYING
@@ -0,0 +1,339 @@
+ GNU GENERAL PUBLIC LICENSE
+ Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The licenses for most software are designed to take away your
+freedom to share and change it. By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users. This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it. (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.) You can apply it to
+your programs, too.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+ To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+ For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have. You must make sure that they, too, receive or can get the
+source code. And you must show them these terms so they know their
+rights.
+
+ We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+ Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software. If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+ Finally, any free program is threatened constantly by software
+patents. We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary. To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ GNU GENERAL PUBLIC LICENSE
+ TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+ 0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License. The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language. (Hereinafter, translation is included without limitation in
+the term "modification".) Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope. The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+ 1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+ 2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+ a) You must cause the modified files to carry prominent notices
+ stating that you changed the files and the date of any change.
+
+ b) You must cause any work that you distribute or publish, that in
+ whole or in part contains or is derived from the Program or any
+ part thereof, to be licensed as a whole at no charge to all third
+ parties under the terms of this License.
+
+ c) If the modified program normally reads commands interactively
+ when run, you must cause it, when started running for such
+ interactive use in the most ordinary way, to print or display an
+ announcement including an appropriate copyright notice and a
+ notice that there is no warranty (or else, saying that you provide
+ a warranty) and that users may redistribute the program under
+ these conditions, and telling the user how to view a copy of this
+ License. (Exception: if the Program itself is interactive but
+ does not normally print such an announcement, your work based on
+ the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole. If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works. But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+ 3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+ a) Accompany it with the complete corresponding machine-readable
+ source code, which must be distributed under the terms of Sections
+ 1 and 2 above on a medium customarily used for software interchange; or,
+
+ b) Accompany it with a written offer, valid for at least three
+ years, to give any third party, for a charge no more than your
+ cost of physically performing source distribution, a complete
+ machine-readable copy of the corresponding source code, to be
+ distributed under the terms of Sections 1 and 2 above on a medium
+ customarily used for software interchange; or,
+
+ c) Accompany it with the information you received as to the offer
+ to distribute corresponding source code. (This alternative is
+ allowed only for noncommercial distribution and only if you
+ received the program in object code or executable form with such
+ an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it. For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable. However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+ 4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License. Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+ 5. You are not required to accept this License, since you have not
+signed it. However, nothing else grants you permission to modify or
+distribute the Program or its derivative works. These actions are
+prohibited by law if you do not accept this License. Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+ 6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions. You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+ 7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all. For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices. Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+ 8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded. In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+ 9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time. Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number. If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation. If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+ 10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission. For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this. Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+ NO WARRANTY
+
+ 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+ 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+ <one line to give the program's name and a brief idea of what it does.>
+ Copyright (C) <year> <name of author>
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+ Gnomovision version 69, Copyright (C) year name of author
+ Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+ This is free software, and you are welcome to redistribute it
+ under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License. Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary. Here is a sample; alter the names:
+
+ Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+ `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+ <signature of Ty Coon>, 1 April 1989
+ Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs. If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library. If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.
diff --git a/README b/README
index f43d89f..fab8c51 100644
--- a/README
+++ b/README
@@ -12,15 +12,9 @@ an appliance Linux solution. Please see the website for overall information.
Usage
-----
-The Baserock builds are controlled by **morphology** files,
-which are build recipes. See below for their syntax. Everything
-in Baserock is built from git commits.
-Morphologies must be committed in git before building. The `morph` tool is
-used to actually run the build. The usual workflow is this:
-
-* put the morphology for an upstream project with its source code
-* put other morphologies in the `morphs` (note plural) repository
-* run `morph` to build stuff
+The Baserock builds are controlled by Baserock definitions files.
+See the documentation at <http://wiki.baserock.org/definitions/current>
+for information on the format.
`morph --help` will provide some information, though a full guide is
really required. Meanwhile a short usage to build a disk image:
@@ -29,7 +23,7 @@ really required. Meanwhile a short usage to build a disk image:
cd workspace
morph checkout baserock:baserock/definitions master
cd master/baserock/baserock/definitions
- morph build base-system-x86_64-generic
+ morph build systems/base-system-x86_64-generic.morph
For deploying you need to create a cluster morphology. Here is an
example to deploy to a raw disk image.
@@ -37,7 +31,7 @@ example to deploy to a raw disk image.
name: foo
kind: cluster
systems:
- - morph: base-system-x86_64-generic
+ - morph: systems/base-system-x86_64-generic.morph
repo: baserock:baserock/definitions
ref: master
deploy:
@@ -49,7 +43,7 @@ example to deploy to a raw disk image.
To deploy it, you only need to run `morph deploy` with the cluster morphology
created:
- morph deploy foo
+ morph deploy foo.morph
You can write a configuration file to avoid having to write options on
the command line every time. Put it in `~/.morph.conf` and make it look
@@ -64,177 +58,6 @@ something like this:
All of the above settings apart from `log` are the defaults, so may be omitted.
-Morphology file syntax
-----------------------
-
-YAML is used for the morphology syntax. For example, to build a chunk:
-
- name: foo
- kind: chunk
- configure-commands:
- - ./configure --prefix="$PREFIX"
- build-commands:
- - make
- test-commands:
- - make check
- install-commands:
- - make DESTDIR="$DESTDIR" install
-
-For all morphologies, use the following fields:
-
-* `name`: the name of the morphology; it must currently match the filename
- (without the `.morph` suffix); **required**
-* `kind`: the kind of thing being built; **required**
-
-For chunks, use the following fields:
-
-
-* `build-system`: if the program is built using a build system known to
- `morph`, you can set this field and avoid having to set the various
- `*-commands` fields; the commands that the build system specifies can
- be overridden; the following build-systems are known:
-
- - `autotools`
- - `python-distutils`
- - `cpan`
- - `cmake`
- - `qmake`
-
- optional
-
-* `pre-configure-commands`: a list of shell commands to run at
- the configuration phase of a build, before the list in `configure-commands`;
- optional
-* `configure-commands`: a list of shell commands to run at the configuraiton
- phase of a build; optional
-* `post-configure-commands`: a list of shell commands to run at
- the configuration phase of a build, after the list in `configure-commands`;
- optional
-
-* `pre-build-commands`: a list of shell commands to run at
- the build phase of a build, before the list in `build-commands`;
- optional
-* `build-commands`: a list of shell commands to run to build (compile) the
- project; optional
-* `post-build-commands`: a list of shell commands to run at
- the build phase of a build, after the list in `build-commands`;
- optional
-
-* `pre-test-commands`: a list of shell commands to run at
- the test phase of a build, before the list in `test-commands`;
- optional
-* `test-commands`: a list of shell commands to run unit tests and other
- non-interactive tests on the built but un-installed project; optional
-* `post-test-commands`: a list of shell commands to run at
- the test phase of a build, after the list in `test-commands`;
- optional
-
-* `pre-install-commands`: a list of shell commands to run at
- the install phase of a build, before the list in `install-commands`;
- optional
-* `install-commands`: a list of shell commands to install the built project;
- the install should go into the directory named in the `DESTDIR` environment
- variable, not the actual system; optional
-* `post-install-commands`: a list of shell commands to run at
- the install phase of a build, after the list in `install-commands`;
- optional
-
-* `max-jobs`: a string to be given to `make` as the argument to the `-j`
- option to specify the maximum number of parallel jobs; the only sensible
- value is `"1"` (including the quotes), to prevent parallel jobs to run
- at all; parallel jobs are only used during the `build-commands` phase,
- since the other phases are often not safe when run in parallel; `morph`
- picks a default value based on the number of CPUs on the host system;
- optional
-
-* `chunks`: a key/value map of lists of regular expressions;
- the key is the name
- of a binary chunk, the regexps match the pathnames that will be
- included in that chunk; the patterns match the pathnames that get installed
- by `install-commands` (the whole path below `DESTDIR`); every file must
- be matched by at least one pattern; by default, a single chunk gets
- created, named according to the morphology, and containing all files;
- optional
-
-For strata, use the following fields:
-
-* `build-depends`: a list of strings, each of which refers to another
- stratum that the current stratum depends on. This list may be omitted
- or empty if the stratum does not depend on anything else.
-* `chunks`: a list of key/value mappings, where each mapping corresponds
- to a chunk to be included in the stratum; the mappings may use the
- following keys: `name` is the chunk's name (may be different from the
- morphology name), `repo` is the repository in which to find (defaults to
- chunk name), `ref` identifies the commit to use (typically a branch
- name, but any tree-ish git accepts is ok), and `morph` is the name
- of the morphology to use and is optional. In addition to these keys,
- each of the sources MUST specify a list of build dependencies using the
- `build-depends` field. This field may be omitted to make the source
- depend on all other chunks that are listed earlier in the `chunks`
- list. The field may be an empty list to indicate that the chunk does
- not depend on anything else in the same stratum. To specify one or
- more chunk dependencies, `build-depends` needs to be set to a list
- that contains the names of chunks that the source depends on in the
- same stratum. These names correspond to the values of the `name`
- fields of the other chunks.
-
-For systems, use the following fields:
-
-* `strata`: a list of names of strata to be included in the system. Unlike
- chunks, the stratum morphs must all be in the same Git repository as the
- system morphology. The value of the `morph` field will be taken as the
- artifact name; if this causes ambiguity then an `alias` may be specified as
- well. **required**
-
-Example chunk (simplified commands):
-
- name: eglibc
- kind: chunk
- configure-commands:
- - mkdir o
- - cd o && ../libc/configure --prefix=/usr
- build-commands:
- - cd o && make
- install-commands:
- - cd o && make install_root="$DESTDIR" install
-
-Example stratum:
-
- name: foundation
- kind: stratum
- chunks:
- - name: fhs-dirs
- repo: upstream:fhs-dirs
- ref: baserock/bootstrap
- build-depends: []
- - name: linux-api-headers
- repo: upstream:linux
- ref: baserock/morph
- build-depends:
- - fhs-dirs
- - name: eglibc
- repo: upstream:eglibc
- ref: baserock/bootstrap
- build-depends:
- - linux-api-headers
- - name: busybox
- repo: upstream:busybox
- ref: baserock/bootstrap
- build-depends:
- - fhs-dirs
- - linux-api-headers
-
-Example system:
-
- name: base
- kind: system
- strata:
- - morph: foundation
- - morph: linux-stratum
-
-Note that currently, unknown keys in morphologies are silently ignored.
-
-
Build environment
-----------------
diff --git a/distbuild/build_controller.py b/distbuild/build_controller.py
index d6f3398..6058862 100644
--- a/distbuild/build_controller.py
+++ b/distbuild/build_controller.py
@@ -116,19 +116,40 @@ def build_step_name(artifact):
return artifact.source.name
-def map_build_graph(artifact, callback):
+def map_build_graph(artifact, callback, components=[]):
+ """Run callback on each artifact in the build graph and return result.
+
+ If components is given, then only look at the components given and
+ their dependencies. Also, return a list of the components after they
+ have had callback called on them.
+
+ """
result = []
+ mapped_components = []
done = set()
- queue = [artifact]
+ if components:
+ queue = list(components)
+ else:
+ queue = [artifact]
while queue:
a = queue.pop()
if a not in done:
result.append(callback(a))
queue.extend(a.source.dependencies)
done.add(a)
- return result
+ if a in components:
+ mapped_components.append(a)
+ return result, mapped_components
+def find_artifacts(components, artifact):
+ found = []
+ for a in artifact.walk():
+ name = a.source.morphology['name']
+ if name in components:
+ found.append(a)
+ return found
+
class BuildController(distbuild.StateMachine):
'''Control one build-request fulfillment.
@@ -165,8 +186,10 @@ class BuildController(distbuild.StateMachine):
spec = [
# state, source, event_class, new_state, callback
('init', self, _Start, 'graphing', self._start_graphing),
- ('init', self._initiator_connection,
- distbuild.InitiatorDisconnect, None, None),
+ ('init', distbuild.InitiatorConnection,
+ distbuild.InitiatorDisconnect, 'init',
+ self._maybe_notify_initiator_disconnected),
+ ('init', self, _Abort, None, None),
('graphing', distbuild.HelperRouter, distbuild.HelperOutput,
'graphing', self._maybe_collect_graph),
@@ -175,16 +198,20 @@ class BuildController(distbuild.StateMachine):
('graphing', self, _GotGraph,
'annotating', self._start_annotating),
('graphing', self, BuildFailed, None, None),
- ('graphing', self._initiator_connection,
- distbuild.InitiatorDisconnect, None, None),
+ ('graphing', distbuild.InitiatorConnection,
+ distbuild.InitiatorDisconnect, 'graphing',
+ self._maybe_notify_initiator_disconnected),
+ ('graphing', self, _Abort, None, None),
('annotating', distbuild.HelperRouter, distbuild.HelperResult,
'annotating', self._maybe_handle_cache_response),
('annotating', self, BuildFailed, None, None),
('annotating', self, _Annotated, 'building',
self._queue_worker_builds),
- ('annotating', self._initiator_connection,
- distbuild.InitiatorDisconnect, None, None),
+ ('annotating', distbuild.InitiatorConnection,
+ distbuild.InitiatorDisconnect, 'annotating',
+ self._maybe_notify_initiator_disconnected),
+ ('annotating', self, _Abort, None, None),
# The exact WorkerConnection that is doing our building changes
# from build to build. We must listen to all messages from all
@@ -314,6 +341,17 @@ class BuildController(distbuild.StateMachine):
distbuild.crash_point()
self._artifact = event.artifact
+ names = self._request['component_names']
+ self._components = find_artifacts(names, self._artifact)
+ failed = False
+ for component in self._components:
+ if component.source.morphology['name'] not in names:
+ logging.debug('Failed to find %s in build graph'
+ % component.filename)
+ failed = True
+ if failed:
+ self.fail('Failed to find all components in %s'
+ % self._artifact.name)
self._helper_id = self._idgen.next()
artifact_names = []
@@ -321,7 +359,9 @@ class BuildController(distbuild.StateMachine):
artifact.state = UNKNOWN
artifact_names.append(artifact.basename())
- map_build_graph(self._artifact, set_state_and_append)
+ _, self._components = map_build_graph(self._artifact,
+ set_state_and_append,
+ self._components)
url = urlparse.urljoin(self._artifact_cache_server, '/1.0/artifacts')
msg = distbuild.message('http-request',
@@ -355,11 +395,20 @@ class BuildController(distbuild.StateMachine):
return
cache_state = json.loads(event.msg['body'])
- map_build_graph(self._artifact, set_status)
+ _, self._components = map_build_graph(self._artifact, set_status,
+ self._components)
self.mainloop.queue_event(self, _Annotated())
- unbuilt = len([a for a in self._artifact.walk() if a.state == UNBUILT])
- total = len([a for _ in self._artifact.walk()])
+ unbuilt = set()
+ for c in self._components:
+ unbuilt.update([a for a in c.walk() if a.state == UNBUILT])
+ unbuilt = len(unbuilt) or len([a for a in self._artifact.walk()
+ if a.state == UNBUILT])
+ total = set()
+ for c in self._components:
+ total.update([a for a in c.walk()])
+ total = len(total) or len([a for _ in self._artifact.walk()])
+
progress = BuildProgress(
self._request['id'],
'Need to build %d artifacts, of %d total' % (unbuilt, total))
@@ -375,22 +424,30 @@ class BuildController(distbuild.StateMachine):
all(a.state == BUILT
for a in artifact.source.dependencies))
- return [a
- for a in map_build_graph(self._artifact, lambda a: a)
- if is_ready_to_build(a)]
+ artifacts, _ = map_build_graph(self._artifact, lambda a: a,
+ self._components)
+ return [a for a in artifacts if is_ready_to_build(a)]
def _queue_worker_builds(self, event_source, event):
distbuild.crash_point()
- if self._artifact.state == BUILT:
- logging.info('Requested artifact is built')
- self.mainloop.queue_event(self, _Built())
- return
+ if not self._components:
+ if self._artifact.state == BUILT:
+ logging.info('Requested artifact is built')
+ self.mainloop.queue_event(self, _Built())
+ return
+
+ else:
+ if not any(c.state != BUILT for c in self._components):
+ logging.info('Requested components are built')
+ self.mainloop.queue_event(self, _Built())
+ return
logging.debug('Queuing more worker-builds to run')
if self.debug_graph_state:
logging.debug('Current state of build graph nodes:')
- for a in map_build_graph(self._artifact, lambda a: a):
+ for a, _ in map_build_graph(self._artifact,
+ lambda a: a, self._components):
logging.debug(' %s state is %s' % (a.name, a.state))
if a.state != BUILT:
for dep in a.dependencies:
@@ -424,7 +481,6 @@ class BuildController(distbuild.StateMachine):
if a.source == artifact.source:
a.state = BUILDING
-
def _maybe_notify_initiator_disconnected(self, event_source, event):
if event.id != self._request['id']:
logging.debug('Heard initiator disconnect with event id %d '
@@ -441,7 +497,7 @@ class BuildController(distbuild.StateMachine):
cancel = BuildCancel(event.id)
self.mainloop.queue_event(BuildController, cancel)
- self.mainloop.queue_event(self, _Abort)
+ self.mainloop.queue_event(self, _Abort())
def _maybe_relay_build_waiting_for_worker(self, event_source, event):
if event.initiator_id != self._request['id']:
@@ -524,7 +580,8 @@ class BuildController(distbuild.StateMachine):
self.mainloop.queue_event(BuildController, progress)
def _find_artifact(self, cache_key):
- artifacts = map_build_graph(self._artifact, lambda a: a)
+ artifacts, _ = map_build_graph(self._artifact, lambda a: a,
+ self._components)
wanted = [a for a in artifacts if a.source.cache_key == cache_key]
if wanted:
return wanted[0]
@@ -559,7 +616,8 @@ class BuildController(distbuild.StateMachine):
# yields all chunk artifacts for the given source
# so we set the state of this source's artifacts
# to BUILT
- map_build_graph(self._artifact, set_state)
+ _, self._components = map_build_graph(self._artifact, set_state,
+ self._components)
self._queue_worker_builds(None, event)
@@ -610,10 +668,19 @@ class BuildController(distbuild.StateMachine):
logging.debug('Notifying initiator of successful build')
baseurl = urlparse.urljoin(
self._artifact_cache_server, '/1.0/artifacts')
- filename = ('%s.%s.%s' %
- (self._artifact.source.cache_key,
- self._artifact.source.morphology['kind'],
- self._artifact.name))
- url = '%s?filename=%s' % (baseurl, urllib.quote(filename))
- finished = BuildFinished(self._request['id'], [url])
+ urls = []
+ for c in self._components:
+ name = ('%s.%s.%s' %
+ (c.source.cache_key,
+ c.source.morphology['kind'],
+ c.name))
+ urls.append('%s?filename=%s' % (baseurl, urllib.quote(name)))
+ if not self._components:
+ name = ('%s.%s.%s' %
+ (self._artifact.source.cache_key,
+ self._artifact.source.morphology['kind'],
+ self._artifact.name))
+ urls.append('%s?filename=%s' % (baseurl, urllib.quote(name)))
+
+ finished = BuildFinished(self._request['id'], urls)
self.mainloop.queue_event(BuildController, finished)
diff --git a/distbuild/initiator.py b/distbuild/initiator.py
index eef4c9e..48299a3 100644
--- a/distbuild/initiator.py
+++ b/distbuild/initiator.py
@@ -54,7 +54,7 @@ def create_build_directory(prefix='build'):
class Initiator(distbuild.StateMachine):
def __init__(self, cm, conn, app, repo_name, ref, morphology,
- original_ref):
+ original_ref, component_names):
distbuild.StateMachine.__init__(self, 'waiting')
self._cm = cm
self._conn = conn
@@ -63,6 +63,10 @@ class Initiator(distbuild.StateMachine):
self._ref = ref
self._morphology = morphology
self._original_ref = original_ref
+ self._component_names = component_names
+ self._partial = False
+ if self._component_names:
+ self._partial = True
self._step_outputs = {}
self.debug_transitions = False
@@ -101,6 +105,8 @@ class Initiator(distbuild.StateMachine):
ref=self._ref,
morphology=self._morphology,
original_ref=self._original_ref,
+ component_names=self._component_names,
+ partial=self._partial,
protocol_version=distbuild.protocol.VERSION
)
self._jm.send(msg)
diff --git a/distbuild/json_router.py b/distbuild/json_router.py
index b8d0ca5..d9c32a9 100644
--- a/distbuild/json_router.py
+++ b/distbuild/json_router.py
@@ -47,7 +47,6 @@ class JsonRouter(distbuild.StateMachine):
def setup(self):
jm = distbuild.JsonMachine(self.conn)
- jm.debug_json = True
self.mainloop.add_state_machine(jm)
spec = [
diff --git a/distbuild/protocol.py b/distbuild/protocol.py
index 73d72d1..268dcbf 100644
--- a/distbuild/protocol.py
+++ b/distbuild/protocol.py
@@ -22,7 +22,7 @@
# time a change is introduced that would break server/initiator compatibility
-VERSION = 1
+VERSION = 2
_required_fields = {
@@ -31,6 +31,7 @@ _required_fields = {
'repo',
'ref',
'morphology',
+ 'partial',
'protocol_version',
],
'build-progress': [
@@ -89,7 +90,8 @@ _required_fields = {
_optional_fields = {
'build-request': [
- 'original_ref'
+ 'original_ref',
+ 'component_names'
]
}
diff --git a/distbuild/worker_build_scheduler.py b/distbuild/worker_build_scheduler.py
index e58059b..8b58117 100644
--- a/distbuild/worker_build_scheduler.py
+++ b/distbuild/worker_build_scheduler.py
@@ -149,8 +149,14 @@ class Jobs(object):
return waiting.pop() if len(waiting) > 0 else None
def __repr__(self):
- return str([job.artifact.basename()
- for (_, job) in self._jobs.iteritems()])
+ items = []
+ for job in self._jobs.itervalues():
+ if job.who is None:
+ state = 'queued'
+ else:
+ state = 'given to %s' % job.who
+ items.append('%s (%s)' % (job.artifact.basename(), state))
+ return str(items)
class _BuildFinished(object):
@@ -400,8 +406,6 @@ class WorkerConnection(distbuild.StateMachine):
self._current_job_exec_response = None
self._current_job_cache_request = None
- self._debug_json = False
-
addr, port = self._conn.getpeername()
name = socket.getfqdn(addr)
self._worker_name = '%s:%s' % (name, port)
@@ -412,6 +416,9 @@ class WorkerConnection(distbuild.StateMachine):
def current_job(self):
return self._current_job
+ def __str__(self):
+ return self.name()
+
def setup(self):
distbuild.crash_point()
@@ -513,10 +520,6 @@ class WorkerConnection(distbuild.StateMachine):
)
self._jm.send(msg)
- if self._debug_json:
- logging.debug('WC: sent to worker %s: %r'
- % (self._worker_name, msg))
-
started = WorkerBuildStepStarted(job.initiators,
job.artifact.source.cache_key, self.name())
diff --git a/morphlib/__init__.py b/morphlib/__init__.py
index 695241c..79e829a 100644
--- a/morphlib/__init__.py
+++ b/morphlib/__init__.py
@@ -37,8 +37,9 @@ __version__ = gitversion.version
# List of architectures that Morph supports
-valid_archs = ['armv7l', 'armv7lhf', 'armv7b', 'testarch',
- 'x86_32', 'x86_64', 'ppc64', 'armv8l64', 'armv8b64']
+valid_archs = ['armv7l', 'armv7lhf', 'armv7b', 'testarch', 'x86_32',
+ 'x86_64', 'ppc64', 'armv8l64', 'armv8b64', 'mips32l',
+ 'mips32b', 'mips64l', 'mips64b']
class Error(cliapp.AppException):
diff --git a/morphlib/app.py b/morphlib/app.py
index c367caf..f5823dd 100644
--- a/morphlib/app.py
+++ b/morphlib/app.py
@@ -20,6 +20,7 @@ import pipes
import sys
import time
import urlparse
+import warnings
import extensions
import morphlib
@@ -59,7 +60,7 @@ class Morph(cliapp.Application):
'show no output unless there is an error')
self.settings.boolean(['help', 'h'],
- 'show this help message and exit')
+ 'show this help message and exit')
self.settings.boolean(['help-all'],
'show help message including hidden subcommands')
@@ -154,6 +155,10 @@ class Morph(cliapp.Application):
'always push temporary build branches to the '
'remote repository',
group=group_build)
+ self.settings.boolean(['partial'],
+ 'only build up to a given chunk',
+ default=False,
+ group=group_build)
self.settings.choice (['local-changes'],
['include', 'ignore'],
'the `build` and `deploy` commands detect '
@@ -229,6 +234,12 @@ class Morph(cliapp.Application):
with morphlib.util.hide_password_environment_variables(os.environ):
cliapp.Application.log_config(self)
+ def pretty_warnings(message, category, filename, lineno,
+ file=None, line=None):
+ return 'WARNING: %s' % (message)
+
+ warnings.formatwarning = pretty_warnings
+
def process_args(self, args):
self.check_time()
@@ -278,8 +289,7 @@ class Morph(cliapp.Application):
sys.exit(0)
tmpdir = self.settings['tempdir']
- for required_dir in (os.path.join(tmpdir, 'chunks'),
- os.path.join(tmpdir, 'staging'),
+ for required_dir in (os.path.join(tmpdir, 'staging'),
os.path.join(tmpdir, 'failed'),
os.path.join(tmpdir, 'deployments'),
self.settings['cachedir']):
@@ -291,11 +301,13 @@ class Morph(cliapp.Application):
def setup_plugin_manager(self):
cliapp.Application.setup_plugin_manager(self)
- self.pluginmgr.locations += os.path.join(
- os.path.dirname(morphlib.__file__), 'plugins')
+ s = os.path.join(os.path.dirname(morphlib.__file__), 'plugins')
+ if not s in self.pluginmgr.locations:
+ self.pluginmgr.locations.append(s)
- s = os.environ.get('MORPH_PLUGIN_PATH', '')
- self.pluginmgr.locations += s.split(':')
+ s = os.environ.get('MORPH_PLUGIN_PATH', '').split(':')
+ for path in s:
+ self.pluginmgr.locations.append(path)
self.hookmgr = cliapp.HookManager()
self.hookmgr.new('new-build-command', cliapp.FilterHook())
@@ -330,7 +342,7 @@ class Morph(cliapp.Application):
* ``error`` should be true when it is an error message
All other keywords are ignored unless embedded in ``msg``.
-
+
The ``self.status_prefix`` string is prepended to the output.
It is set to the empty string by default.
@@ -385,7 +397,7 @@ class Morph(cliapp.Application):
self._write_status(self._commandline_as_message(argv, args))
# Log the environment.
- prev = getattr(self, 'prev_env', {})
+ prev = getattr(self, 'prev_env', os.environ)
morphlib.util.log_environment_changes(self, kwargs['env'], prev)
self.prev_env = kwargs['env']
diff --git a/morphlib/buildbranch.py b/morphlib/buildbranch.py
index 80cecd7..2a2530b 100644
--- a/morphlib/buildbranch.py
+++ b/morphlib/buildbranch.py
@@ -103,7 +103,7 @@ class BuildBranch(object):
in index.get_uncommitted_changes()]
if not changed:
continue
- add_cb(gd=gd, build_ref=gd, changed=changed)
+ add_cb(gd=gd, build_ref=build_ref, changed=changed)
changes_made = True
index.add_files_from_working_tree(changed)
return changes_made
@@ -303,9 +303,8 @@ def pushed_build_branch(bb, loader, changes_need_pushing, name, email,
build_uuid, status):
with contextlib.closing(bb) as bb:
def report_add(gd, build_ref, changed):
- status(msg='Adding uncommitted changes '\
- 'in %(dirname)s to %(ref)s',
- dirname=gd.dirname, ref=build_ref, chatty=True)
+ status(msg='Creating temporary branch in %(dirname)s '\
+ 'named %(ref)s', dirname=gd.dirname, ref=build_ref)
changes_made = bb.add_uncommitted_changes(add_cb=report_add)
unpushed = any(bb.get_unpushed_branches())
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index c83abca..cab3839 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -74,7 +74,8 @@ class BuildCommand(object):
This includes creating the directories on disk if they are missing.
'''
- return morphlib.util.new_artifact_caches(self.app.settings)
+ return morphlib.util.new_artifact_caches(
+ self.app.settings, status_cb=self.app.status)
def new_repo_caches(self):
return morphlib.util.new_repo_caches(self.app)
@@ -119,7 +120,10 @@ class BuildCommand(object):
root_kind = root_artifact.source.morphology['kind']
if root_kind != 'system':
raise morphlib.Error(
- 'Building a %s directly is not supported' % root_kind)
+ 'In order to build this %s directly, please give the filename '
+ 'of the system which contains it, and the name of the %s. '
+ 'See `morph build --help` for more information.'
+ % (root_kind, root_kind))
def _validate_architecture(self, root_artifact):
'''Perform the validation between root and target architectures.'''
@@ -271,7 +275,8 @@ class BuildCommand(object):
def build_in_order(self, root_artifact):
'''Build everything specified in a build order.'''
- self.app.status(msg='Building a set of sources')
+ self.app.status(msg='Starting build of %(name)s',
+ name=root_artifact.source.name)
build_env = root_artifact.build_env
ordered_sources = list(self.get_ordered_sources(root_artifact.walk()))
old_prefix = self.app.status_prefix
@@ -487,32 +492,13 @@ class BuildCommand(object):
if artifact.source.build_mode == 'bootstrap':
if not self.in_same_stratum(artifact.source, target_source):
continue
+
self.app.status(
msg='Installing chunk %(chunk_name)s from cache %(cache)s',
chunk_name=artifact.name,
cache=artifact.source.cache_key[:7],
chatty=True)
- chunk_cache_dir = os.path.join(self.app.settings['tempdir'],
- 'chunks')
- artifact_checkout = os.path.join(
- chunk_cache_dir, os.path.basename(artifact.basename()) + '.d')
- if not os.path.exists(artifact_checkout):
- self.app.status(
- msg='Checking out %(chunk)s from cache.',
- chunk=artifact.name
- )
- temp_checkout = os.path.join(self.app.settings['tempdir'],
- artifact.basename())
- try:
- self.lac.get(artifact, temp_checkout)
- except BaseException:
- shutil.rmtree(temp_checkout)
- raise
- # TODO: This rename is not concurrency safe if two builds are
- # extracting the same chunk, one build will fail because
- # the other renamed its tempdir here first.
- os.rename(temp_checkout, artifact_checkout)
- staging_area.install_artifact(artifact, artifact_checkout)
+ staging_area.install_artifact(self.lac, artifact)
if target_source.build_mode == 'staging':
morphlib.builder.ldconfig(self.app.runcmd, staging_area.dirname)
@@ -540,7 +526,8 @@ class InitiatorBuildCommand(BuildCommand):
self.app.settings['push-build-branches'] = True
super(InitiatorBuildCommand, self).__init__(app)
- def build(self, repo_name, ref, filename, original_ref=None):
+ def build(self, repo_name, ref, filename, original_ref=None,
+ component_names=[]):
'''Initiate a distributed build on a controller'''
distbuild.add_crash_conditions(self.app.settings['crash-condition'])
@@ -551,7 +538,8 @@ class InitiatorBuildCommand(BuildCommand):
self.app.status(msg='Starting distributed build')
loop = distbuild.MainLoop()
- args = [repo_name, ref, filename, original_ref or ref]
+ args = [repo_name, ref, filename, original_ref or ref,
+ component_names]
cm = distbuild.InitiatorConnectionMachine(self.app,
self.addr,
self.port,
diff --git a/morphlib/buildenvironment.py b/morphlib/buildenvironment.py
index 6ec82d4..266510f 100644
--- a/morphlib/buildenvironment.py
+++ b/morphlib/buildenvironment.py
@@ -114,16 +114,30 @@ class BuildEnvironment():
# than leaving it up to individual morphologies.
if arch == 'x86_32':
cpu = 'i686'
+ abi = ''
+ elif arch.startswith('armv7'):
+ cpu = arch
+ abi = 'eabi'
elif arch == 'armv8l64': # pragma: no cover
cpu = 'aarch64'
+ abi = ''
elif arch == 'armv8b64': # pragma: no cover
cpu = 'aarch64_be'
+ abi = ''
+ elif arch == 'mips64b': # pragma: no cover
+ cpu = 'mips64'
+ abi = 'abi64'
+ elif arch == 'mips64l': # pragma: no cover
+ cpu = 'mips64el'
+ abi = 'abi64'
+ elif arch == 'mips32b': # pragma: no cover
+ cpu = 'mips'
+ abi = ''
+ elif arch == 'mips32l': # pragma: no cover
+ cpu = 'mipsel'
+ abi = ''
else:
cpu = arch
-
- if arch.startswith('armv7'):
- abi = 'eabi'
- else:
abi = ''
env['TARGET'] = cpu + '-baserock-linux-gnu' + abi
diff --git a/morphlib/builder.py b/morphlib/builder.py
index e5b891b..b0c95bb 100644
--- a/morphlib/builder.py
+++ b/morphlib/builder.py
@@ -558,16 +558,32 @@ class SystemBuilder(BuilderBase): # pragma: no cover
self.save_build_times()
return self.source.artifacts.itervalues()
+ def load_stratum(self, stratum_artifact):
+ '''Load a stratum from the local artifact cache.
+
+ Returns a list of ArtifactCacheReference instances for the chunks
+ contained in the stratum.
+
+ '''
+ cache = self.local_artifact_cache
+ with open(cache.get(stratum_artifact), 'r') as stratum_file:
+ try:
+ artifact_list = json.load(stratum_file,
+ encoding='unicode-escape')
+ except ValueError as e:
+ raise cliapp.AppException(
+ 'Corruption detected: %s while loading %s' %
+ (e, cache.artifact_filename(stratum_artifact)))
+ return [ArtifactCacheReference(a) for a in artifact_list]
+
def unpack_one_stratum(self, stratum_artifact, target):
'''Unpack a single stratum into a target directory'''
cache = self.local_artifact_cache
- with open(cache.get(stratum_artifact), 'r') as stratum_file:
- artifact_list = json.load(stratum_file, encoding='unicode-escape')
- for chunk in (ArtifactCacheReference(a) for a in artifact_list):
- self.app.status(msg='Checkout chunk %(basename)s',
- basename=chunk.basename(), chatty=True)
- cache.get(chunk, target)
+ for chunk in self.load_stratum(stratum_artifact):
+ self.app.status(msg='Checkout chunk %(basename)s',
+ basename=chunk.basename(), chatty=True)
+ cache.get(chunk, target)
target_metadata = os.path.join(
target, 'baserock', '%s.meta' % stratum_artifact.name)
@@ -590,11 +606,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover
# download the chunk artifacts if necessary
for stratum_artifact in self.source.dependencies:
- stratum_path = self.local_artifact_cache.get(
- stratum_artifact)
- with open(stratum_path, 'r') as stratum:
- chunks = [ArtifactCacheReference(c)
- for c in json.load(stratum)]
+ chunks = self.load_stratum(stratum_artifact)
download_depends(chunks,
self.local_artifact_cache,
self.remote_artifact_cache)
diff --git a/morphlib/cachedrepo.py b/morphlib/cachedrepo.py
index 2363904..b41ba86 100644
--- a/morphlib/cachedrepo.py
+++ b/morphlib/cachedrepo.py
@@ -123,6 +123,26 @@ class CachedRepo(object):
'''
return self._gitdir.read_file(filename, ref)
+ def tags_containing_sha1(self, ref): # pragma: no cover
+ '''Check whether given sha1 is contained in any tags
+
+ Raises a gitdir.InvalidRefError if the ref is not found in the
+ repository. Raises gitdir.ExpectedSha1Error if the ref is not
+ a sha1.
+
+ '''
+ return self._gitdir.tags_containing_sha1(ref)
+
+ def branches_containing_sha1(self, ref): # pragma: no cover
+ '''Check whether given sha1 is contained in any branches
+
+ Raises a gitdir.InvalidRefError if the ref is not found in the
+ repository. Raises gitdir.ExpectedSha1Error if the ref is not
+ a sha1.
+
+ '''
+ return self._gitdir.branches_containing_sha1(ref)
+
def list_files(self, ref, recurse=True): # pragma: no cover
'''Return filenames found in the tree pointed to by the given ref.
diff --git a/morphlib/exts/fstab.configure b/morphlib/exts/fstab.configure
index 3bbc910..b9154ee 100755
--- a/morphlib/exts/fstab.configure
+++ b/morphlib/exts/fstab.configure
@@ -1,5 +1,6 @@
-#!/usr/bin/python
-# Copyright (C) 2013,2015 Codethink Limited
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright © 2013-2015 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -19,21 +20,9 @@
import os
import sys
+import morphlib
-def asciibetical(strings):
+envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('FSTAB_')}
- def key(s):
- return [ord(c) for c in s]
-
- return sorted(strings, key=key)
-
-
-fstab_filename = os.path.join(sys.argv[1], 'etc', 'fstab')
-
-fstab_vars = asciibetical(x for x in os.environ if x.startswith('FSTAB_'))
-with open(fstab_filename, 'a') as f:
- for var in fstab_vars:
- f.write('%s\n' % os.environ[var])
-
-os.chown(fstab_filename, 0, 0)
-os.chmod(fstab_filename, 0644)
+conf_file = os.path.join(sys.argv[1], 'etc/fstab')
+morphlib.util.write_from_dict(conf_file, envvars)
diff --git a/morphlib/exts/hosts.configure b/morphlib/exts/hosts.configure
new file mode 100755
index 0000000..6b068d0
--- /dev/null
+++ b/morphlib/exts/hosts.configure
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright © 2015 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# =*= License: GPL-2 =*=
+
+
+import os
+import sys
+import socket
+
+import morphlib
+
+def validate(var, line):
+ xs = line.split()
+ if len(xs) == 0:
+ raise morphlib.Error("`%s: %s': line is empty" % (var, line))
+
+ ip = xs[0]
+ hostnames = xs[1:]
+
+ if len(hostnames) == 0:
+ raise morphlib.Error("`%s: %s': missing hostname" % (var, line))
+
+ family = socket.AF_INET6 if ':' in ip else socket.AF_INET
+
+ try:
+ socket.inet_pton(family, ip)
+ except socket.error:
+ raise morphlib.Error("`%s: %s' invalid ip" % (var, ip))
+
+envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('HOSTS_')}
+
+conf_file = os.path.join(sys.argv[1], 'etc/hosts')
+morphlib.util.write_from_dict(conf_file, envvars, validate)
diff --git a/morphlib/exts/install-files.configure b/morphlib/exts/install-files.configure
index 58cf373..c297024 100755
--- a/morphlib/exts/install-files.configure
+++ b/morphlib/exts/install-files.configure
@@ -30,6 +30,12 @@ import shlex
import shutil
import stat
+try:
+ import jinja2
+ jinja_available = True
+except ImportError:
+ jinja_available = False
+
class InstallFilesConfigureExtension(cliapp.Application):
def process_args(self, args):
@@ -48,18 +54,20 @@ class InstallFilesConfigureExtension(cliapp.Application):
self.install_entry(entry, manifest_dir, target_root)
def install_entry(self, entry, manifest_root, target_root):
- m = re.match('(overwrite )?([0-7]+) ([0-9]+) ([0-9]+) (\S+)', entry)
+ m = re.match('(template )?(overwrite )?'
+ '([0-7]+) ([0-9]+) ([0-9]+) (\S+)', entry)
if m:
- overwrite = m.group(1)
- mode = int(m.group(2), 8) # mode is octal
- uid = int(m.group(3))
- gid = int(m.group(4))
- path = m.group(5)
+ template = m.group(1)
+ overwrite = m.group(2)
+ mode = int(m.group(3), 8) # mode is octal
+ uid = int(m.group(4))
+ gid = int(m.group(5))
+ path = m.group(6)
else:
raise cliapp.AppException('Invalid manifest entry, '
- 'format: [overwrite] <octal mode> <uid decimal> <gid decimal> '
- '<filename>')
+ 'format: [template] [overwrite] '
+ '<octal mode> <uid decimal> <gid decimal> <filename>')
dest_path = os.path.join(target_root, './' + path)
if stat.S_ISDIR(mode):
@@ -91,8 +99,22 @@ class InstallFilesConfigureExtension(cliapp.Application):
raise cliapp.AppException('File already exists at %s'
% dest_path)
else:
- shutil.copyfile(os.path.join(manifest_root, './' + path),
- dest_path)
+ if template:
+ if not jinja_available:
+ raise cliapp.AppException(
+ "Failed to install template file `%s': "
+ 'install-files templates require jinja2'
+ % path)
+
+ loader = jinja2.FileSystemLoader(manifest_root)
+ env = jinja2.Environment(loader=loader,
+ keep_trailing_newline=True)
+
+ env.get_template(path).stream(os.environ).dump(dest_path)
+ else:
+ shutil.copyfile(os.path.join(manifest_root, './' + path),
+ dest_path)
+
os.chown(dest_path, uid, gid)
os.chmod(dest_path, mode)
diff --git a/morphlib/exts/kvm.check b/morphlib/exts/kvm.check
index 62d7645..67cb3d3 100755
--- a/morphlib/exts/kvm.check
+++ b/morphlib/exts/kvm.check
@@ -47,6 +47,7 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension):
self.check_no_existing_libvirt_vm(ssh_host, vm_name)
self.check_extra_disks_exist(ssh_host, self.parse_attach_disks())
self.check_virtual_networks_are_started(ssh_host)
+ self.check_host_has_virtinstall(ssh_host)
def check_and_parse_location(self, location):
'''Check and parse the location argument to get relevant data.'''
@@ -129,14 +130,22 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension):
def name(nic_entry):
if ',' in nic_entry:
- # NETWORK_NAME,mac=12:34,model=e1000...
- return nic_entry[:nic_entry.find(',')]
+ # network=NETWORK_NAME,mac=12:34,model=e1000...
+ return nic_entry[:nic_entry.find(',')].lstrip('network=')
else:
- return nic_entry # NETWORK_NAME
+ return nic_entry.lstrip('network=') # NETWORK_NAME
if 'NIC_CONFIG' in os.environ:
nics = os.environ['NIC_CONFIG'].split()
+ for n in nics:
+ if not (n.startswith('network=')
+ or n.startswith('bridge=')
+ or n == 'user'):
+ raise cliapp.AppException('malformed NIC_CONFIG: %s\n'
+ " (expected 'bridge=BRIDGE' 'network=NAME'"
+ " or 'user')" % n)
+
# --network bridge= is used to specify a bridge
# --network user is used to specify a form of NAT
# (see the virt-install(1) man page)
@@ -148,5 +157,13 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension):
for network in networks:
check_virtual_network_is_started(network)
+ def check_host_has_virtinstall(self, ssh_host):
+ try:
+ cliapp.ssh_runcmd(ssh_host, ['which', 'virt-install'])
+ except cliapp.AppException:
+ raise cliapp.AppException(
+ 'virt-install does not seem to be installed on host %s'
+ % ssh_host)
+
KvmPlusSshCheckExtension().run()
diff --git a/morphlib/exts/simple-network.configure b/morphlib/exts/simple-network.configure
index 6111332..1ba94e8 100755
--- a/morphlib/exts/simple-network.configure
+++ b/morphlib/exts/simple-network.configure
@@ -27,6 +27,7 @@ for DHCP
import os
import sys
+import errno
import cliapp
import morphlib
@@ -80,12 +81,14 @@ class SimpleNetworkConfigurationExtension(cliapp.Application):
"""
file_path = os.path.join(args[0], "etc", "systemd", "network",
"10-dhcp.network")
- try:
- os.rename(file_path, file_path + ".morph")
- self.status(msg="Renaming networkd file from systemd chunk: %(f)s \
- to %(f)s.morph", f=file_path)
- except OSError:
- pass
+
+ if os.path.isfile(file_path):
+ try:
+ os.rename(file_path, file_path + ".morph")
+ self.status(msg="Renaming networkd file from systemd chunk: \
+ %(f)s to %(f)s.morph", f=file_path)
+ except OSError:
+ pass
def generate_default_network_config(self, args):
"""Generate default network config: DHCP in all the interfaces"""
@@ -106,7 +109,11 @@ class SimpleNetworkConfigurationExtension(cliapp.Application):
"""Generate /etc/network/interfaces file"""
iface_file = self.generate_iface_file(stanzas)
- with open(os.path.join(args[0], "etc/network/interfaces"), "w") as f:
+
+ directory_path = os.path.join(args[0], "etc", "network")
+ self.make_sure_path_exists(directory_path)
+ file_path = os.path.join(directory_path, "interfaces")
+ with open(file_path, "w") as f:
f.write(iface_file)
def generate_iface_file(self, stanzas):
@@ -147,10 +154,12 @@ class SimpleNetworkConfigurationExtension(cliapp.Application):
if iface_file is None:
continue
- path = os.path.join(args[0], "etc", "systemd", "network",
- "%s-%s.network" % (i, stanza['name']))
+ directory_path = os.path.join(args[0], "etc", "systemd", "network")
+ self.make_sure_path_exists(directory_path)
+ file_path = os.path.join(directory_path,
+ "%s-%s.network" % (i, stanza['name']))
- with open(path, "w") as f:
+ with open(file_path, "w") as f:
f.write(iface_file)
def generate_networkd_file(self, stanza):
@@ -252,6 +261,16 @@ class SimpleNetworkConfigurationExtension(cliapp.Application):
return output_stanza
+ def make_sure_path_exists(self, path):
+ try:
+ os.makedirs(path)
+ except OSError as e:
+ if e.errno == errno.EEXIST and os.path.isdir(path):
+ pass
+ else:
+ raise SimpleNetworkError("Unable to create directory '%s'"
+ % path)
+
def status(self, **kwargs):
'''Provide status output.
diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py
index 03640a2..1c28672 100644
--- a/morphlib/gitdir.py
+++ b/morphlib/gitdir.py
@@ -681,6 +681,32 @@ class GitDirectory(object):
if not morphlib.git.is_valid_sha1(string):
raise ExpectedSha1Error(string)
+ def _check_ref_exists(self, ref):
+ self._rev_parse('%s^{commit}' % ref)
+
+ def _gitcmd_output_list(self, *args):
+ output = morphlib.git.gitcmd(self._runcmd, *args)
+ separated = [l.strip() for l in output.splitlines()]
+ prefix = '* '
+ for i, l in enumerate(separated):
+ if l.startswith(prefix):
+ separated[i] = l[len(prefix):]
+ return separated
+
+ def tags_containing_sha1(self, ref): # pragma: no cover
+ self._check_is_sha1(ref)
+ self._check_ref_exists(ref)
+
+ args = ['tag', '--contains', ref]
+ return self._gitcmd_output_list(*args)
+
+ def branches_containing_sha1(self, ref):
+ self._check_is_sha1(ref)
+ self._check_ref_exists(ref)
+
+ args = ['branch', '--contains', ref]
+ return self._gitcmd_output_list(*args)
+
def _update_ref(self, ref_args, message):
args = ['update-ref']
# No test coverage, since while this functionality is useful,
diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py
index a6e1921..f606dfe 100644
--- a/morphlib/gitdir_tests.py
+++ b/morphlib/gitdir_tests.py
@@ -95,6 +95,38 @@ class GitDirectoryTests(unittest.TestCase):
self.assertIsInstance(gitdir.get_index(), morphlib.gitindex.GitIndex)
+class GitDirectoryAnchoredRefTests(unittest.TestCase):
+
+ def setUp(self):
+ self.tempdir = tempfile.mkdtemp()
+ self.dirname = os.path.join(self.tempdir, 'foo')
+ os.mkdir(self.dirname)
+ gd = morphlib.gitdir.init(self.dirname)
+ with open(os.path.join(self.dirname, 'test_file.morph'), "w") as f:
+ f.write('dummy morphology text')
+ morphlib.git.gitcmd(gd._runcmd, 'add', '.')
+ morphlib.git.gitcmd(gd._runcmd, 'commit', '-m', 'Initial commit')
+
+ def tearDown(self):
+ shutil.rmtree(self.tempdir)
+
+ def test_ref_anchored_in_branch(self):
+ gd = morphlib.gitdir.GitDirectory(self.dirname)
+ output = morphlib.git.gitcmd(gd._runcmd, 'rev-parse', 'HEAD')
+ ref = output.strip()
+
+ self.assertEqual(len(gd.branches_containing_sha1(ref)), 1)
+ self.assertEqual(gd.branches_containing_sha1(ref)[0], 'master')
+
+ def test_ref_not_anchored_in_branch(self):
+ gd = morphlib.gitdir.GitDirectory(self.dirname)
+ output = morphlib.git.gitcmd(gd._runcmd, 'rev-parse', 'HEAD')
+ ref = output.strip()
+
+ morphlib.git.gitcmd(gd._runcmd, 'commit', '--amend', '-m',
+ 'New commit message')
+ self.assertEqual(len(gd.branches_containing_sha1(ref)), 0)
+
class GitDirectoryContentsTests(unittest.TestCase):
def setUp(self):
diff --git a/morphlib/localartifactcache.py b/morphlib/localartifactcache.py
index e6695c4..9e3c9e7 100644
--- a/morphlib/localartifactcache.py
+++ b/morphlib/localartifactcache.py
@@ -132,8 +132,9 @@ class LocalArtifactCache(object):
CacheInfo = collections.namedtuple('CacheInfo', ('artifacts', 'mtime'))
contents = collections.defaultdict(lambda: CacheInfo(set(), 0))
for filename in self.cachefs.walkfiles():
- cachekey = filename[:63]
- artifact = filename[65:]
+ if filename.startswith('/repo'): # pragma: no cover
+ continue
+ cachekey, artifact = filename.split('.', 1)
artifacts, max_mtime = contents[cachekey]
artifacts.add(artifact)
art_info = self.cachefs.getinfo(filename)
diff --git a/morphlib/ostreeartifactcache.py b/morphlib/ostreeartifactcache.py
index ea659c8..230460f 100644
--- a/morphlib/ostreeartifactcache.py
+++ b/morphlib/ostreeartifactcache.py
@@ -15,8 +15,10 @@
import collections
+import contextlib
import logging
import os
+import stat
import shutil
import tarfile
import tempfile
@@ -27,26 +29,48 @@ from gi.repository import GLib
import morphlib
from morphlib.artifactcachereference import ArtifactCacheReference
+
+class NotCachedError(morphlib.Error):
+
+ def __init__(self, ref):
+ self.msg = 'Failed to checkout %s from the artifact cache.' % ref
+
+
class OSTreeArtifactCache(object):
"""Class to provide the artifact cache API using an OSTree repo."""
- def __init__(self, cachedir, mode):
+ def __init__(self, cachedir, mode='bare', status_cb=None):
repo_dir = os.path.join(cachedir, 'repo')
self.repo = morphlib.ostree.OSTreeRepo(repo_dir, mode=mode)
self.cachedir = cachedir
+ self.status_cb = status_cb
+
+ def status(self, *args, **kwargs):
+ if self.status_cb is not None:
+ self.status_cb(*args, **kwargs)
+ @contextlib.contextmanager
def _get_file_from_remote(self, artifact, remote, metadata_name=None):
if metadata_name:
handle = remote.get_artifact_metadata(artifact, metadata_name)
+ self.status(
+ msg='Downloading %(name)s %(metadata_name)s as a file.',
+ chatty=True, name=artifact.basename(),
+ metadata_name=metadata_name)
else:
handle = remote.get(artifact)
- fd, path = tempfile.mkstemp()
- with open(path, 'w+') as temp:
- shutil.copyfileobj(handle, temp)
- return path
+ self.status(
+ msg='Downloading %(name)s as a tarball.', chatty=True,
+ name=artifact.basename())
+
+ try:
+ temporary_download = tempfile.NamedTemporaryFile(dir=self.cachedir)
+ shutil.copyfileobj(handle, temporary_download)
+ yield temporary_download.name
+ finally:
+ temporary_download.close()
def _get_artifact_cache_name(self, artifact):
- logging.debug('LAC: %s' % artifact.basename())
cache_key, kind, name = artifact.basename().split('.', 2)
suffix = name.split('-')[-1]
return '%s-%s' % (cache_key, suffix)
@@ -58,11 +82,13 @@ class OSTreeArtifactCache(object):
contents of directory should be the contents of the artifact.
"""
+ cache_key, kind, name = artifact.basename().split('.', 2)
ref = self._get_artifact_cache_name(artifact)
- subject = artifact.name
+ subject = name
try:
- logging.debug('Committing %s to artifact cache at %s.' %
- (subject, ref))
+ self.status(
+ msg='Committing %(subject)s to artifact cache at %(ref)s.',
+ chatty=True, subject=subject, ref=ref)
self.repo.commit(subject, directory, ref)
except GLib.GError as e:
logging.debug('OSTree raised an exception: %s' % e)
@@ -77,44 +103,71 @@ class OSTreeArtifactCache(object):
else:
filename = self.artifact_filename(artifact)
shutil.copy(location, filename)
- os.remove(location)
+
+ def _remove_device_nodes(self, path):
+ for dirpath, dirnames, filenames in os.walk(path):
+ for f in filenames:
+ filepath = os.path.join(dirpath, f)
+ mode = os.lstat(filepath).st_mode
+ if stat.S_ISBLK(mode) or stat.S_ISCHR(mode):
+ logging.debug('Removing device node %s from artifact' %
+ filepath)
+ os.remove(filepath)
+
+ def _copy_metadata_from_remote(self, artifact, remote):
+ """Copy a metadata file from a remote cache."""
+ a, name = artifact.basename().split('.', 1)
+ with self._get_file_from_remote(ArtifactCacheReference(a),
+ remote, name) as location:
+ self.put_non_ostree_artifact(ArtifactCacheReference(a),
+ location, name)
def copy_from_remote(self, artifact, remote):
- """Get 'artifact' from remote artifact cache and store it locally."""
+ """Get 'artifact' from remote artifact cache and store it locally.
+
+ This takes an Artifact object and a RemoteArtifactCache. Note that
+ `remote` here is not the same as a `remote` for and OSTree repo.
+
+ """
if remote.method == 'tarball':
- logging.debug('Downloading artifact tarball for %s.' %
- artifact.name)
- location = self._get_file_from_remote(artifact, remote)
- try:
- tempdir = tempfile.mkdtemp()
- with tarfile.open(name=location) as tf:
- tf.extractall(path=tempdir)
+ with self._get_file_from_remote(artifact, remote) as location:
try:
+ cache_key, kind, name = artifact.basename().split('.', 2)
+ except ValueError:
+ # We can't split the name properly, it must be metadata!
+ self._copy_metadata_from_remote(artifact, remote)
+ return
+
+ if kind == 'stratum':
+ self.put_non_ostree_artifact(artifact, location)
+ return
+ try:
+ tempdir = tempfile.mkdtemp(dir=self.cachedir)
+ with tarfile.open(name=location) as tf:
+ tf.extractall(path=tempdir)
+ self._remove_device_nodes(tempdir)
self.put(tempdir, artifact)
+ except tarfile.ReadError:
+ # Reading the tarball failed, and we expected a
+ # tarball artifact. Something must have gone
+ # wrong.
+ raise
finally:
- os.remove(location)
shutil.rmtree(tempdir)
- except tarfile.ReadError:
- # Reading the artifact as a tarball failed, so it must be a
- # single file (for example a stratum artifact).
- self.put_non_ostree_artifact(artifact, location)
elif remote.method == 'ostree':
- logging.debug('Pulling artifact for %s from remote.' %
- artifact.basename())
+ self.status(msg='Pulling artifact for %(name)s from remote.',
+ chatty=True, name=artifact.basename())
try:
ref = self._get_artifact_cache_name(artifact)
- except Exception:
+ except ValueError:
# if we can't split the name properly, we must want metadata
- a, name = artifact.basename().split('.', 1)
- location = self._get_file_from_remote(
- ArtifactCacheReference(a), remote, name)
- self.put_non_ostree_artifact(artifact, location, name)
+ self._copy_metadata_from_remote(artifact, remote)
return
if artifact.basename().split('.', 2)[1] == 'stratum':
- location = self._get_file_from_remote(artifact, remote)
- self.put_non_ostree_artifact(artifact, location)
+ with self._get_file_from_remote(artifact, remote) as location:
+ self.put_non_ostree_artifact(artifact, location)
return
try:
@@ -126,7 +179,7 @@ class OSTreeArtifactCache(object):
raise cliapp.AppException('Failed to pull %s from remote '
'cache.' % ref)
- def get(self, artifact, directory=None, status=lambda a: a):
+ def get(self, artifact, directory=None):
"""Checkout an artifact from the repo and return its location."""
cache_key, kind, name = artifact.basename().split('.', 2)
if kind == 'stratum':
@@ -136,11 +189,13 @@ class OSTreeArtifactCache(object):
ref = self._get_artifact_cache_name(artifact)
try:
self.repo.checkout(ref, directory)
+ # We need to update the mtime and atime of the ref file in the
+ # repository so that we can decide which refs were least recently
+ # accessed when doing `morph gc`.
self.repo.touch_ref(ref)
except GLib.GError as e:
logging.debug('OSTree raised an exception: %s' % e)
- raise cliapp.AppException('Failed to checkout %s from artifact '
- 'cache.' % ref)
+ raise NotCachedError(ref)
return directory
def list_contents(self):
@@ -173,16 +228,26 @@ class OSTreeArtifactCache(object):
self.repo.prune()
def has(self, artifact):
- cachekey, kind, name = artifact.basename().split('.', 2)
- logging.debug('OSTreeArtifactCache: got %s, %s, %s' %
- (cachekey, kind, name))
+ try:
+ cachekey, kind, name = artifact.basename().split('.', 2)
+ except ValueError:
+ # We couldn't split the basename properly, we must want metadata
+ cachekey, name = artifact.basename().split('.', 1)
+ if self.has_artifact_metadata(artifact, name):
+ return True
+ else:
+ return False
+
+ if kind == 'stratum':
+ if self._has_file(self.artifact_filename(artifact)):
+ return True
+ else:
+ return False
+
sha = self.repo.resolve_rev(self._get_artifact_cache_name(artifact))
if sha:
self.repo.touch_ref(self._get_artifact_cache_name(artifact))
return True
- if kind == 'stratum' and \
- self._has_file(self.artifact_filename(artifact)):
- return True
return False
def get_artifact_metadata(self, artifact, name):
diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py
index cdbb303..08589ea 100644
--- a/morphlib/plugins/branch_and_merge_plugin.py
+++ b/morphlib/plugins/branch_and_merge_plugin.py
@@ -623,7 +623,6 @@ class BranchAndMergePlugin(cliapp.Plugin):
smd = morphlib.systemmetadatadir.SystemMetadataDir(path)
metadata = smd.values()
- logging.debug(metadata)
systems = [md for md in metadata
if 'kind' in md and md['kind'] == 'system']
diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py
index 2cc395f..e5b3585 100644
--- a/morphlib/plugins/build_plugin.py
+++ b/morphlib/plugins/build_plugin.py
@@ -13,26 +13,39 @@
# with this program. If not, see <http://www.gnu.org/licenses/>.
-import cliapp
+import collections
import contextlib
import uuid
import logging
+import cliapp
+
import morphlib
+class ComponentNotInSystemError(morphlib.Error):
+
+ def __init__(self, components, system):
+ components = ', '.join(components)
+ self.msg = ('Components %s are not in %s. Ensure you provided '
+ 'component names rather than filenames.'
+ % (components, system))
+
+
class BuildPlugin(cliapp.Plugin):
def enable(self):
self.app.add_subcommand('build-morphology', self.build_morphology,
- arg_synopsis='(REPO REF FILENAME)...')
+ arg_synopsis='REPO REF FILENAME '
+ '[COMPONENT...]')
self.app.add_subcommand('build', self.build,
- arg_synopsis='SYSTEM')
+ arg_synopsis='SYSTEM [COMPONENT...]')
self.app.add_subcommand('distbuild-morphology',
self.distbuild_morphology,
- arg_synopsis='SYSTEM')
+ arg_synopsis='REPO REF FILENAME '
+ '[COMPONENT...]')
self.app.add_subcommand('distbuild', self.distbuild,
- arg_synopsis='SYSTEM')
+ arg_synopsis='SYSTEM [COMPONENT...]')
self.use_distbuild = False
def disable(self):
@@ -46,6 +59,8 @@ class BuildPlugin(cliapp.Plugin):
* `REPO` is a git repository URL.
* `REF` is a branch or other commit reference in that repository.
* `FILENAME` is a morphology filename at that ref.
+ * `COMPONENT...` is the names of one or more chunks or strata to
+ build. If none are given the the system at FILENAME is built.
See 'help distbuild' and 'help build-morphology' for more information.
@@ -54,10 +69,15 @@ class BuildPlugin(cliapp.Plugin):
addr = self.app.settings['controller-initiator-address']
port = self.app.settings['controller-initiator-port']
+ self.use_distbuild = True
build_command = morphlib.buildcommand.InitiatorBuildCommand(
self.app, addr, port)
- for repo_name, ref, filename in self.app.itertriplets(args):
- build_command.build(repo_name, ref, filename)
+ repo, ref, filename = args[0:3]
+ filename = morphlib.util.sanitise_morphology_path(filename)
+ component_names = [morphlib.util.sanitise_morphology_path(name)
+ for name in args[3:]]
+ self.start_build(repo, ref, build_command, filename,
+ component_names)
def distbuild(self, args):
'''Distbuild a system image in the current system branch
@@ -65,6 +85,8 @@ class BuildPlugin(cliapp.Plugin):
Command line arguments:
* `SYSTEM` is the name of the system to build.
+ * `COMPONENT...` is the names of one or more chunks or strata to
+ build. If none are given then SYSTEM is built.
This command launches a distributed build, to use this command
you must first set up a distbuild cluster.
@@ -92,6 +114,8 @@ class BuildPlugin(cliapp.Plugin):
* `REPO` is a git repository URL.
* `REF` is a branch or other commit reference in that repository.
* `FILENAME` is a morphology filename at that ref.
+ * `COMPONENT...` is the names of one or more chunks or strata to
+ build. If none are given then the system at FILENAME is built.
You probably want `morph build` instead. However, in some
cases it is more convenient to not have to create a Morph
@@ -104,8 +128,14 @@ class BuildPlugin(cliapp.Plugin):
Example:
- morph build-morphology baserock:baserock/definitions \
- master devel-system-x86_64-generic.morph
+ morph build-morphology baserock:baserock/definitions \\
+ master systems/devel-system-x86_64-generic.morph
+
+ Partial build example:
+
+ morph build-morphology baserock:baserock/definitions \\
+ master systems/devel-system-x86_64-generic.morph \\
+ build-essential
'''
@@ -117,15 +147,21 @@ class BuildPlugin(cliapp.Plugin):
self.app.settings['cachedir-min-space'])
build_command = morphlib.buildcommand.BuildCommand(self.app)
- for repo_name, ref, filename in self.app.itertriplets(args):
- build_command.build(repo_name, ref, filename)
+ repo, ref, filename = args[0:3]
+ filename = morphlib.util.sanitise_morphology_path(filename)
+ component_names = [morphlib.util.sanitise_morphology_path(name)
+ for name in args[3:]]
+ self.start_build(repo, ref, build_command, filename,
+ component_names)
def build(self, args):
'''Build a system image in the current system branch
Command line arguments:
- * `SYSTEM` is the name of the system to build.
+ * `SYSTEM` is the filename of the system to build.
+ * `COMPONENT...` is the names of one or more chunks or strata to
+ build. If this is not given then the SYSTEM is built.
This builds a system image, and any of its components that
need building. The system name is the basename of the system
@@ -145,14 +181,14 @@ class BuildPlugin(cliapp.Plugin):
Example:
- morph build devel-system-x86_64-generic.morph
+ morph build systems/devel-system-x86_64-generic.morph
- '''
+ Partial build example:
- if len(args) != 1:
- raise cliapp.AppException('morph build expects exactly one '
- 'parameter: the system to build')
+ morph build systems/devel-system-x86_64-generic.morph \\
+ build-essential
+ '''
# Raise an exception if there is not enough space
morphlib.util.check_disk_available(
self.app.settings['tempdir'],
@@ -165,6 +201,7 @@ class BuildPlugin(cliapp.Plugin):
system_filename = morphlib.util.sanitise_morphology_path(args[0])
system_filename = sb.relative_to_root_repo(system_filename)
+ component_names = args[1:]
logging.debug('System branch is %s' % sb.root_directory)
@@ -178,11 +215,14 @@ class BuildPlugin(cliapp.Plugin):
build_command = morphlib.buildcommand.BuildCommand(self.app)
if self.app.settings['local-changes'] == 'include':
- self._build_with_local_changes(build_command, sb, system_filename)
+ self._build_with_local_changes(build_command, sb, system_filename,
+ component_names)
else:
- self._build_local_commit(build_command, sb, system_filename)
+ self._build_local_commit(build_command, sb, system_filename,
+ component_names)
- def _build_with_local_changes(self, build_command, sb, system_filename):
+ def _build_with_local_changes(self, build_command, sb, system_filename,
+ component_names):
'''Construct a branch including user's local changes, and build that.
It is often a slow process to check all repos in the system branch for
@@ -199,9 +239,12 @@ class BuildPlugin(cliapp.Plugin):
email = morphlib.git.get_user_email(self.app.runcmd)
build_ref_prefix = self.app.settings['build-ref-prefix']
- self.app.status(msg='Starting build %(uuid)s', uuid=build_uuid)
+ self.app.status(msg='Looking for uncommitted changes (pass '
+ '--local-changes=ignore to skip)')
+
self.app.status(msg='Collecting morphologies involved in '
'building %(system)s from %(branch)s',
+ chatty=True,
system=system_filename,
branch=sb.system_branch_name)
@@ -211,10 +254,11 @@ class BuildPlugin(cliapp.Plugin):
name=name, email=email, build_uuid=build_uuid,
status=self.app.status)
with pbb as (repo, commit, original_ref):
- build_command.build(repo, commit, system_filename,
- original_ref=original_ref)
+ self.start_build(repo, commit, build_command, system_filename,
+ component_names, original_ref=original_ref)
- def _build_local_commit(self, build_command, sb, system_filename):
+ def _build_local_commit(self, build_command, sb, system_filename,
+ component_names):
'''Build whatever commit the user has checked-out locally.
This ignores any uncommitted changes. Also, if the user has a commit
@@ -242,4 +286,47 @@ class BuildPlugin(cliapp.Plugin):
definitions_repo = morphlib.gitdir.GitDirectory(definitions_repo_path)
commit = definitions_repo.resolve_ref_to_commit(ref)
- build_command.build(root_repo_url, commit, system_filename)
+ self.start_build(root_repo_url, commit, build_command,
+ system_filename, component_names)
+
+ def _find_artifacts(self, names, root_artifact):
+ found = collections.OrderedDict()
+ not_found = names
+ for a in root_artifact.walk():
+ name = a.source.morphology['name']
+ if name in names and name not in found:
+ found[name] = a
+ not_found.remove(name)
+ return found, not_found
+
+ def start_build(self, repo, commit, bc, system_filename,
+ component_names, original_ref=None):
+ '''Actually run the build.
+
+ If a set of components was given, only build those. Otherwise,
+ build the whole system.
+
+ '''
+ if self.use_distbuild:
+ bc.build(repo, commit, system_filename,
+ original_ref=original_ref,
+ component_names=component_names)
+ return
+
+ self.app.status(msg='Deciding on task order')
+ srcpool = bc.create_source_pool(repo, commit, system_filename)
+ bc.validate_sources(srcpool)
+ root = bc.resolve_artifacts(srcpool)
+ if not component_names:
+ component_names = [root.source.name]
+ components, not_found = self._find_artifacts(component_names, root)
+ if not_found:
+ raise ComponentNotInSystemError(not_found, system_filename)
+
+ for name, component in components.iteritems():
+ component.build_env = root.build_env
+ bc.build_in_order(component)
+ self.app.status(msg='%(kind)s %(name)s is cached at %(path)s',
+ kind=component.source.morphology['kind'],
+ name=name,
+ path=bc.lac.artifact_filename(component))
diff --git a/morphlib/plugins/certify_plugin.py b/morphlib/plugins/certify_plugin.py
new file mode 100644
index 0000000..10fc19a
--- /dev/null
+++ b/morphlib/plugins/certify_plugin.py
@@ -0,0 +1,140 @@
+# Copyright (C) 2014-2015 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# This plugin is used as part of the Baserock automated release process.
+#
+# See: <http://wiki.baserock.org/guides/release-process> for more information.
+
+import warnings
+
+import cliapp
+import morphlib
+
+class CertifyPlugin(cliapp.Plugin):
+
+ def enable(self):
+ self.app.add_subcommand(
+ 'certify', self.certify,
+ arg_synopsis='REPO REF MORPH [MORPH]...')
+
+ def disable(self):
+ pass
+
+ def certify(self, args):
+ '''Certify that any given system definition is reproducable.
+
+ Command line arguments:
+
+ * `REPO` is a git repository URL.
+ * `REF` is a branch or other commit reference in that repository.
+ * `MORPH` is a system morphology name at that ref.
+
+ '''
+
+ if len(args) < 3:
+ raise cliapp.AppException(
+ 'Wrong number of arguments to certify command '
+ '(see help)')
+
+ repo, ref = args[0], args[1]
+ system_filenames = map(morphlib.util.sanitise_morphology_path,
+ args[2:])
+
+ self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app)
+ self.resolver = morphlib.artifactresolver.ArtifactResolver()
+
+ for system_filename in system_filenames:
+ self.certify_system(repo, ref, system_filename)
+
+ def certify_system(self, repo, ref, system_filename):
+ '''Certify reproducibility of system.'''
+
+ self.app.status(
+ msg='Creating source pool for %s' % system_filename, chatty=True)
+ source_pool = morphlib.sourceresolver.create_source_pool(
+ self.lrc, self.rrc, repo, ref, system_filename,
+ cachedir=self.app.settings['cachedir'],
+ update_repos = not self.app.settings['no-git-update'],
+ status_cb=self.app.status)
+
+ self.app.status(
+ msg='Resolving artifacts for %s' % system_filename, chatty=True)
+ root_artifacts = self.resolver.resolve_root_artifacts(source_pool)
+
+ def find_artifact_by_name(artifacts_list, filename):
+ for a in artifacts_list:
+ if a.source.filename == filename:
+ return a
+ raise ValueError
+
+ system_artifact = find_artifact_by_name(root_artifacts,
+ system_filename)
+
+ self.app.status(
+ msg='Computing cache keys for %s' % system_filename, chatty=True)
+ build_env = morphlib.buildenvironment.BuildEnvironment(
+ self.app.settings, system_artifact.source.morphology['arch'])
+ ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env)
+
+ aliases = self.app.settings['repo-alias']
+ resolver = morphlib.repoaliasresolver.RepoAliasResolver(aliases)
+
+ certified = True
+
+ for source in set(a.source for a in system_artifact.walk()):
+ source.cache_key = ckc.compute_key(source)
+ source.cache_id = ckc.get_cache_id(source)
+
+ if source.morphology['kind'] != 'chunk':
+ continue
+
+ name = source.morphology['name']
+ ref = source.original_ref
+
+ # Test that chunk has a sha1 ref
+ # TODO: Could allow either sha1 or existent tag.
+ if not morphlib.git.is_valid_sha1(ref):
+ warnings.warn('Chunk "{}" has non-sha1 ref: "{}"\n'
+ .format(name, ref))
+ certified = False
+
+ # Ensure we have a cache of the repo
+ if not self.lrc.has_repo(source.repo_name):
+ self.lrc.cache_repo(source.repo_name)
+
+ cached = self.lrc.get_repo(source.repo_name)
+
+ # Test that sha1 ref is anchored in a tag or branch,
+ # and thus not a candidate for removal on `git gc`.
+ if (morphlib.git.is_valid_sha1(ref) and
+ not len(cached.tags_containing_sha1(ref)) and
+ not len(cached.branches_containing_sha1(ref))):
+ warnings.warn('Chunk "{}" has unanchored ref: "{}"\n'
+ .format(name, ref))
+ certified = False
+
+ # Test that chunk repo is on trove-host
+ pull_url = resolver.pull_url(source.repo_name)
+ if self.app.settings['trove-host'] not in pull_url:
+ warnings.warn('Chunk "{}" has repo not on trove-host: "{}"\n'
+ .format(name, pull_url))
+ certified = False
+
+ if certified:
+ print('=> Reproducibility certification PASSED for\n {}'
+ .format(system_filename))
+ else:
+ print('=> Reproducibility certification FAILED for\n {}'
+ .format(system_filename))
diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py
index 79609cb..9bec564 100644
--- a/morphlib/plugins/cross-bootstrap_plugin.py
+++ b/morphlib/plugins/cross-bootstrap_plugin.py
@@ -27,7 +27,7 @@ echo "Generated by Morph version %s\n"
set -eu
-export PATH=$PATH:/tools/bin:/tools/sbin
+export PATH=/usr/bin:/bin:/usr/sbin:/sbin:/tools/bin:/tools/sbin
export SRCDIR=/src
''' % morphlib.__version__
diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py
index efe6735..3c19553 100644
--- a/morphlib/plugins/deploy_plugin.py
+++ b/morphlib/plugins/deploy_plugin.py
@@ -13,6 +13,7 @@
# with this program. If not, see <http://www.gnu.org/licenses/>.
+import collections
import json
import logging
import os
@@ -27,6 +28,14 @@ import morphlib
from morphlib.artifactcachereference import ArtifactCacheReference
+class NotYetBuiltError(morphlib.Error):
+
+ def __init__(self, name):
+ self.msg = ('Deployment failed as %s is not yet built.\n'
+ 'Please ensure the system is built before deployment.'
+ % name)
+
+
class DeployPlugin(cliapp.Plugin):
def enable(self):
@@ -385,7 +394,7 @@ class DeployPlugin(cliapp.Plugin):
name=name, email=email, build_uuid=build_uuid,
status=self.app.status)
with pbb as (repo, commit, original_ref):
- self.deploy_cluster(build_command, cluster_morphology,
+ self.deploy_cluster(sb, build_command, cluster_morphology,
root_repo_dir, repo, commit, env_vars,
deployments)
else:
@@ -398,6 +407,11 @@ class DeployPlugin(cliapp.Plugin):
deployments)
self.app.status(msg='Finished deployment')
+ if self.app.settings['partial']:
+ self.app.status(msg='WARNING: This was a partial deployment. '
+ 'Configuration extensions have not been '
+ 'run. Applying the result to an existing '
+ 'system may not have reproducible results.')
def validate_deployment_options(
self, env_vars, all_deployments, all_subsystems):
@@ -415,21 +429,54 @@ class DeployPlugin(cliapp.Plugin):
'Variable referenced a non-existent deployment '
'name: %s' % var)
- def deploy_cluster(self, build_command, cluster_morphology, root_repo_dir,
- repo, commit, env_vars, deployments):
+ def deploy_cluster(self, sb, build_command, cluster_morphology,
+ root_repo_dir, repo, commit, env_vars, deployments):
# Create a tempdir for this deployment to work in
deploy_tempdir = tempfile.mkdtemp(
dir=os.path.join(self.app.settings['tempdir'], 'deployments'))
try:
for system in cluster_morphology['systems']:
- self.deploy_system(build_command, deploy_tempdir,
+ self.deploy_system(sb, build_command, deploy_tempdir,
root_repo_dir, repo, commit, system,
env_vars, deployments,
parent_location='')
finally:
shutil.rmtree(deploy_tempdir)
- def deploy_system(self, build_command, deploy_tempdir,
+ def _sanitise_morphology_paths(self, paths, sb):
+ sanitised_paths = []
+ for path in paths:
+ path = morphlib.util.sanitise_morphology_path(path)
+ sanitised_paths.append(sb.relative_to_root_repo(path))
+ return sanitised_paths
+
+ def _find_artifacts(self, filenames, root_artifact):
+ found = collections.OrderedDict()
+ not_found = filenames
+ for a in root_artifact.walk():
+ if a.source.filename in filenames and a.source.name not in found:
+ found[a.source.name] = a
+ not_found.remove(a.source.filename)
+ return found, not_found
+
+ def _validate_partial_deployment(self, deployment_type,
+ artifact, component_names):
+ supported_types = ('tar', 'sysroot')
+ if deployment_type not in supported_types:
+ raise cliapp.AppException('Not deploying %s, --partial was '
+ 'set and partial deployment only '
+ 'supports %s deployments.' %
+ (artifact.source.name,
+ ', '.join(supported_types)))
+ components, not_found = self._find_artifacts(component_names,
+ artifact)
+ if not_found:
+ raise cliapp.AppException('Components %s not found in system %s.' %
+ (', '.join(not_found),
+ artifact.source.name))
+ return components
+
+ def deploy_system(self, sb, build_command, deploy_tempdir,
root_repo_dir, build_repo, ref, system, env_vars,
deployment_filter, parent_location):
sys_ids = set(system['deploy'].iterkeys())
@@ -475,6 +522,12 @@ class DeployPlugin(cliapp.Plugin):
raise morphlib.Error('"type" is undefined '
'for system "%s"' % system_id)
+ components = self._sanitise_morphology_paths(
+ deploy_params.get('partial-deploy-components', []), sb)
+ if self.app.settings['partial']:
+ components = self._validate_partial_deployment(
+ deployment_type, artifact, components)
+
location = final_env.pop('location', None)
if not location:
raise morphlib.Error('"location" is undefined '
@@ -488,9 +541,10 @@ class DeployPlugin(cliapp.Plugin):
root_repo_dir,
ref, artifact,
deployment_type,
- location, final_env)
+ location, final_env,
+ components=components)
for subsystem in system.get('subsystems', []):
- self.deploy_system(build_command, deploy_tempdir,
+ self.deploy_system(sb, build_command, deploy_tempdir,
root_repo_dir, build_repo,
ref, subsystem, env_vars, [],
parent_location=system_tree)
@@ -542,13 +596,27 @@ class DeployPlugin(cliapp.Plugin):
pass
def checkout_stratum(self, path, artifact, lac, rac):
+ """Pull the chunks in a stratum, and checkout them into `path`.
+
+ This reads a stratum artifact and pulls the chunks it contains from
+ the remote into the local artifact cache if they are not already
+ cached locally. Each of these chunks is then checked out into `path`.
+
+ Also download the stratum metadata into the local cache, then place
+ it in the /baserock directory of the system checkout indicated by
+ `path`.
+
+ If any of the chunks have not been cached either locally or remotely,
+ a morphlib.remoteartifactcache.GetError is raised.
+
+ """
with open(lac.get(artifact), 'r') as stratum:
chunks = [ArtifactCacheReference(c) for c in json.load(stratum)]
morphlib.builder.download_depends(chunks, lac, rac)
for chunk in chunks:
self.app.status(msg='Checkout chunk %(name)s.',
name=chunk.basename(), chatty=True)
- lac.get(chunk, path, self.app.status)
+ lac.get(chunk, path)
metadata = os.path.join(path, 'baserock', '%s.meta' % artifact.name)
with lac.get_artifact_metadata(artifact, 'meta') as meta_src:
@@ -556,14 +624,94 @@ class DeployPlugin(cliapp.Plugin):
shutil.copyfileobj(meta_src, meta_dst)
def checkout_strata(self, path, artifact, lac, rac):
+ """Pull the dependencies of `artifact` and checkout them into `path`.
+
+ This assumes that `artifact` is a system artifact. If any of the
+ dependencies aren't cached remotely or locally, this raises a
+ morphlib.remoteartifactcache.GetError.
+
+ """
deps = artifact.source.dependencies
morphlib.builder.download_depends(deps, lac, rac)
for stratum in deps:
self.checkout_stratum(path, stratum, lac, rac)
morphlib.builder.ldconfig(self.app.runcmd, path)
+ def checkout_system(self, build_command, artifact, path):
+ """Checkout a system into `path`.
+
+ This checks out each of the strata into the directory given by `path`,
+ then checks out the system artifact into the same directory. This uses
+ OSTree's `union` checkout mode to overwrite duplicate files but not
+ need an empty directory. Artifacts which aren't cached locally are
+ fetched from the remote cache.
+
+ Raises a NotYetBuiltError if either the system artifact or any of the
+ chunk artifacts in the strata which make up the system aren't cached
+ either locally or remotely.
+
+ """
+ # Check if the system artifact is in the local or remote cache.
+ # If it isn't, we don't need to bother checking out strata before
+ # we fail.
+ if not (build_command.lac.has(artifact)
+ or build_command.rac.has(artifact)):
+ raise NotYetBuiltError(artifact.name)
+
+ # Checkout the strata involved in the artifact into a tempdir
+ self.app.status(msg='Checking out strata in system')
+ try:
+ self.checkout_strata(path, artifact,
+ build_command.lac, build_command.rac)
+
+ self.app.status(msg='Checking out system for configuration')
+ build_command.cache_artifacts_locally([artifact])
+ build_command.lac.get(artifact, path)
+ except (morphlib.ostreeartifactcache.NotCachedError,
+ morphlib.remoteartifactcache.GetError):
+ raise NotYetBuiltError(artifact.name)
+
+ self.app.status(
+ msg='System checked out at %(system_tree)s',
+ system_tree=path)
+
+ def checkout_components(self, bc, components, path):
+ if not components:
+ raise cliapp.AppException('Deployment failed as no components '
+ 'were specified for deployment and '
+ '--partial was set.')
+ for name, artifact in components.iteritems():
+ deps = artifact.source.dependencies
+ morphlib.builder.download_depends(deps, bc.lac, bc.rac)
+ for dep in deps:
+ if dep.source.morphology['kind'] == 'stratum':
+ self.checkout_stratum(path, dep, bc.lac, bc.rac)
+ elif dep.source.morphology['kind'] == 'chunk':
+ self.app.status(msg='Checkout chunk %(name)s.',
+ name=dep.basename(), chatty=True)
+ bc.lac.get(dep, path)
+ if artifact.source.morphology['kind'] == 'stratum':
+ self.checkout_stratum(path, artifact, bc.lac, bc.rac)
+ elif artifact.source.morphology['kind'] == 'chunk':
+ self.app.status(msg='Checkout chunk %(name)s.',
+ name=name, chatty=True)
+ bc.lac.get(artifact, path)
+ self.app.status(
+ msg='Components %(components)s checkout out at %(path)s',
+ components=', '.join(components), path=path)
+
def setup_deploy(self, build_command, deploy_tempdir, root_repo_dir, ref,
- artifact, deployment_type, location, env):
+ artifact, deployment_type, location, env, components=[]):
+ """Checkout the artifact, create metadata and return the location.
+
+ This checks out the system into a temporary directory, and then mounts
+ this temporary directory alongside a different temporary directory
+ using a union filesystem. This allows changes to be made without
+ touching the checked out artifacts. The deployment metadata file is
+ created and then the directory at which the two temporary directories
+ are mounted is returned.
+
+ """
# deployment_type, location and env are only used for saving metadata
deployment_dir = tempfile.mkdtemp(dir=deploy_tempdir)
@@ -583,26 +731,11 @@ class DeployPlugin(cliapp.Plugin):
deploy_tree = os.path.join(deployment_dir,
'overlay-deploy-%s' % artifact.name)
try:
- # Checkout the strata involved in the artifact into a tempdir
- self.app.status(msg='Checking out strata in system')
- self.checkout_strata(system_tree, artifact,
- build_command.lac, build_command.rac)
-
- self.app.status(msg='Checking out system for configuration')
- if build_command.lac.has(artifact):
- build_command.lac.get(artifact, system_tree)
- elif build_command.rac.has(artifact):
- build_command.cache_artifacts_locally([artifact])
- build_command.lac.get(artifact, system_tree)
+ if self.app.settings['partial']:
+ self.checkout_components(build_command, components,
+ system_tree)
else:
- raise cliapp.AppException('Deployment failed as system is'
- ' not yet built.\nPlease ensure'
- ' the system is built before'
- ' deployment.')
-
- self.app.status(
- msg='System checked out at %(system_tree)s',
- system_tree=system_tree)
+ self.checkout_system(build_command, artifact, system_tree)
union_filesystem = self.app.settings['union-filesystem']
morphlib.fsutils.overlay_mount(self.app.runcmd,
@@ -625,10 +758,7 @@ class DeployPlugin(cliapp.Plugin):
except Exception:
if deploy_tree and os.path.exists(deploy_tree):
morphlib.fsutils.unmount(self.app.runcmd, deploy_tree)
- shutil.rmtree(deploy_tree)
- shutil.rmtree(system_tree)
- shutil.rmtree(overlay_dir)
- shutil.rmtree(work_dir)
+ shutil.rmtree(deployment_dir)
raise
def run_deploy_commands(self, deploy_tempdir, env, artifact, root_repo_dir,
@@ -640,15 +770,19 @@ class DeployPlugin(cliapp.Plugin):
try:
# Run configuration extensions.
- self.app.status(msg='Configure system')
- names = artifact.source.morphology['configuration-extensions']
- for name in names:
- self._run_extension(
- root_repo_dir,
- name,
- '.configure',
- [system_tree],
- env)
+ if not self.app.settings['partial']:
+ self.app.status(msg='Configure system')
+ names = artifact.source.morphology['configuration-extensions']
+ for name in names:
+ self._run_extension(
+ root_repo_dir,
+ name,
+ '.configure',
+ [system_tree],
+ env)
+ else:
+ self.app.status(msg='WARNING: Not running configuration '
+ 'extensions as --partial is set!')
# Run write extension.
self.app.status(msg='Writing to device')
@@ -665,7 +799,7 @@ class DeployPlugin(cliapp.Plugin):
shutil.rmtree(deploy_private_tempdir)
def _report_extension_stdout(self, line):
- self.app.status(msg=line.replace('%s', '%%'))
+ self.app.status(msg=line.replace('%', '%%'))
def _report_extension_stderr(self, error_list):
def cb(line):
error_list.append(line)
@@ -699,7 +833,7 @@ class DeployPlugin(cliapp.Plugin):
raise cliapp.AppException(message)
def create_metadata(self, system_artifact, root_repo_dir, deployment_type,
- location, env):
+ location, env, components=[]):
'''Deployment-specific metadata.
The `build` and `deploy` operations must be from the same ref, so full
@@ -731,6 +865,9 @@ class DeployPlugin(cliapp.Plugin):
'commit': morphlib.gitversion.commit,
'version': morphlib.gitversion.version,
},
+ 'partial': self.app.settings['partial'],
}
+ if self.app.settings['partial']:
+ meta['partial-components'] = components
return meta
diff --git a/morphlib/plugins/gc_plugin.py b/morphlib/plugins/gc_plugin.py
index 8b5dc4c..54c1b43 100644
--- a/morphlib/plugins/gc_plugin.py
+++ b/morphlib/plugins/gc_plugin.py
@@ -157,10 +157,9 @@ class GCPlugin(cliapp.Plugin):
self.app.status(msg='Removing source %(cachekey)s',
cachekey=cachekey, chatty=True)
lac.remove(cachekey)
+ lac.prune()
removed += 1
- lac.prune()
-
if sufficient_free():
self.app.status(msg='Made sufficient space in %(cache_path)s '
'after removing %(removed)d sources',
diff --git a/morphlib/plugins/get_chunk_details_plugin.py b/morphlib/plugins/get_chunk_details_plugin.py
new file mode 100644
index 0000000..842b4af
--- /dev/null
+++ b/morphlib/plugins/get_chunk_details_plugin.py
@@ -0,0 +1,79 @@
+# Copyright (C) 2015 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import cliapp
+import morphlib
+
+class GetChunkDetailsPlugin(cliapp.Plugin):
+
+ def enable(self):
+ self.app.add_subcommand(
+ 'get-chunk-details', self.get_chunk_details,
+ arg_synopsis='[STRATUM] CHUNK')
+
+ def disable(self):
+ pass
+
+ def get_chunk_details(self, args):
+ '''Print out details for the given chunk
+
+ Command line arguments:
+
+ * `STRATUM` is the stratum to search for chunk (optional).
+ * `CHUNK` is the component to obtain a URL for.
+
+ '''
+
+ stratum_name = None
+
+ if len(args) == 1:
+ chunk_name = args[0]
+ elif len(args) == 2:
+ stratum_name = args[0]
+ chunk_name = args[1]
+ else:
+ raise cliapp.AppException(
+ 'Wrong number of arguments to get-chunk-details command '
+ '(see help)')
+
+ sb = morphlib.sysbranchdir.open_from_within('.')
+ loader = morphlib.morphloader.MorphologyLoader()
+
+ aliases = self.app.settings['repo-alias']
+ self.resolver = morphlib.repoaliasresolver.RepoAliasResolver(aliases)
+
+ found = 0
+ for morph in sb.load_all_morphologies(loader):
+ if morph['kind'] == 'stratum':
+ if (stratum_name == None or
+ morph['name'] == stratum_name):
+ for chunk in morph['chunks']:
+ if chunk['name'] == chunk_name:
+ found = found + 1
+ self._print_chunk_details(chunk, morph)
+
+ if found == 0:
+ if stratum_name == None:
+ print('Chunk `{}` not found'
+ .format(chunk_name))
+ else:
+ print('Chunk `{}` not found in stratum `{}`'
+ .format(chunk_name, stratum_name))
+
+ def _print_chunk_details(self, chunk, morph):
+ repo = self.resolver.pull_url(chunk['repo'])
+ print('In stratum {}:'.format(morph['name']))
+ print(' Chunk: {}'.format(chunk['name']))
+ print(' Repo: {}'.format(repo))
+ print(' Ref: {}'.format(chunk['ref']))
diff --git a/morphlib/plugins/ostree_artifacts_plugin.py b/morphlib/plugins/ostree_artifacts_plugin.py
new file mode 100644
index 0000000..eedcd1e
--- /dev/null
+++ b/morphlib/plugins/ostree_artifacts_plugin.py
@@ -0,0 +1,169 @@
+# Copyright (C) 2015 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+import collections
+import fs
+import os
+
+import cliapp
+
+import morphlib
+from morphlib.artifactcachereference import ArtifactCacheReference
+
+
+class NoCacheError(morphlib.Error):
+
+ def __init__(self, cachedir):
+ self.msg = ("Expected artifact cache directory %s doesn't exist.\n"
+ "No existing cache to convert!" % cachedir)
+
+
+class ComponentNotInSystemError(morphlib.Error):
+
+ def __init__(self, components, system):
+ components = ', '.join(components)
+ self.msg = ('Components %s are not in %s. Ensure you provided '
+ 'component names rather than filenames.'
+ % (components, system))
+
+
+class OSTreeArtifactsPlugin(cliapp.Plugin):
+
+ def enable(self):
+ self.app.add_subcommand('convert-local-cache', self.convert_cache,
+ arg_synopsis='[DELETE]')
+ self.app.add_subcommand('query-cache', self.query_cache,
+ arg_synopsis='SYSTEM NAME...')
+
+ def disable(self):
+ pass
+
+ def convert_cache(self, args):
+ """Convert a local tarball cache into an OSTree cache.
+
+ Command line arguments:
+
+ * DELETE: This is an optional argument, which if given as "delete"
+ will cause tarball artifacts to be removed once they are converted.
+
+ This command will extract all the tarball artifacts in your local
+ artifact cache and store them in an OSTree repository in that
+ artifact cache. This will be quicker than redownloading all that
+ content from a remote cache server, but may still be time consuming
+ if your cache is large.
+
+ """
+ delete = False
+ if args:
+ if args[0] == 'delete':
+ delete = True
+
+ artifact_cachedir = os.path.join(self.app.settings['cachedir'],
+ 'artifacts')
+ if not os.path.exists(artifact_cachedir):
+ raise NoCacheError(artifact_cachedir)
+
+ tarball_cache = morphlib.localartifactcache.LocalArtifactCache(
+ fs.osfs.OSFS(artifact_cachedir))
+ ostree_cache = morphlib.ostreeartifactcache.OSTreeArtifactCache(
+ artifact_cachedir, mode=self.app.settings['ostree-repo-mode'],
+ status_cb=self.app.status)
+
+ cached_artifacts = []
+ for cachekey, artifacts, last_used in tarball_cache.list_contents():
+ for artifact in artifacts:
+ basename = '.'.join((cachekey.lstrip('/'), artifact))
+ cached_artifacts.append(ArtifactCacheReference(basename))
+
+ # Set the method property of the tarball cache to allow us to
+ # treat it like a RemoteArtifactCache.
+ tarball_cache.method = 'tarball'
+
+ for artifact in cached_artifacts:
+ if not ostree_cache.has(artifact):
+ try:
+ cache_key, kind, name = artifact.basename().split('.', 2)
+ if kind in ('system', 'stratum'):
+ # System artifacts are quick to recreate now, and
+ # stratum artifacts are still stored in the same way.
+ continue
+ except ValueError:
+ # We must have metadata, which doesn't need converting
+ continue
+ self.app.status(msg='Converting %(name)s',
+ name=artifact.basename())
+ ostree_cache.copy_from_remote(artifact, tarball_cache)
+ if delete:
+ os.remove(tarball_cache.artifact_filename(artifact))
+
+ def _find_artifacts(self, names, root_artifact):
+ found = collections.OrderedDict()
+ not_found = list(names)
+ for a in root_artifact.walk():
+ name = a.source.morphology['name']
+ if name in names and name not in found:
+ found[name] = [a]
+ if name in not_found:
+ not_found.remove(name)
+ elif name in names:
+ found[name].append(a)
+ if name in not_found:
+ not_found.remove(name)
+ return found, not_found
+
+ def query_cache(self, args):
+ """Check if the cache contains an artifact.
+
+ Command line arguments:
+
+ * `SYSTEM` is the filename of the system containing the components
+ to be looked for.
+ * `NAME...` is the name of one or more components to look for.
+
+ """
+ if not args:
+ raise cliapp.AppException('You must provide at least a system '
+ 'filename.\nUsage: `morph query-cache '
+ 'SYSTEM [NAME...]`')
+ ws = morphlib.workspace.open('.')
+ sb = morphlib.sysbranchdir.open_from_within('.')
+
+ system_filename = morphlib.util.sanitise_morphology_path(args[0])
+ system_filename = sb.relative_to_root_repo(system_filename)
+ component_names = args[1:]
+
+ bc = morphlib.buildcommand.BuildCommand(self.app)
+ repo = sb.get_config('branch.root')
+ ref = sb.get_config('branch.name')
+
+ definitions_repo_path = sb.get_git_directory_name(repo)
+ definitions_repo = morphlib.gitdir.GitDirectory(definitions_repo_path)
+ commit = definitions_repo.resolve_ref_to_commit(ref)
+
+ srcpool = bc.create_source_pool(repo, commit, system_filename)
+ bc.validate_sources(srcpool)
+ root = bc.resolve_artifacts(srcpool)
+ if not component_names:
+ component_names = [root.source.name]
+ components, not_found = self._find_artifacts(component_names, root)
+ if not_found:
+ raise ComponentNotInSystemError(not_found, system_filename)
+
+ for name, artifacts in components.iteritems():
+ for component in artifacts:
+ if bc.lac.has(component):
+ print bc.lac._get_artifact_cache_name(component)
+ else:
+ print '%s is not cached' % name
diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py
index d2b47d3..771e81e 100644
--- a/morphlib/sourceresolver.py
+++ b/morphlib/sourceresolver.py
@@ -31,7 +31,7 @@ tree_cache_filename = 'trees.cache.pickle'
buildsystem_cache_size = 10000
buildsystem_cache_filename = 'detected-chunk-buildsystems.cache.pickle'
-not_supported_versions = []
+supported_versions = [0, 1, 2]
class PickleCacheManager(object): # pragma: no cover
'''Cache manager for PyLRU that reads and writes to Pickle files.
@@ -90,12 +90,26 @@ class MorphologyNotFoundError(SourceResolverError): # pragma: no cover
SourceResolverError.__init__(
self, "Couldn't find morphology: %s" % filename)
+
+class MorphologyReferenceNotFoundError(SourceResolverError): # pragma: no cover
+ def __init__(self, filename, reference_file):
+ SourceResolverError.__init__(self,
+ "Couldn't find morphology: %s "
+ "referenced in %s"
+ % (filename, reference_file))
+
+
class UnknownVersionError(SourceResolverError): # pragma: no cover
def __init__(self, version):
SourceResolverError.__init__(
self, "Definitions format version %s is not supported" % version)
+class InvalidVersionFileError(SourceResolverError): #pragma: no cover
+ def __init__(self):
+ SourceResolverError.__init__(self, "invalid VERSION file")
+
+
class SourceResolver(object):
'''Provides a way of resolving the set of sources for a given system.
@@ -286,7 +300,7 @@ class SourceResolver(object):
loader = morphlib.morphloader.MorphologyLoader()
text = self._get_file_contents(reponame, sha1, filename)
- morph = loader.load_from_string(text)
+ morph = loader.load_from_string(text, filename)
if morph is not None:
self._resolved_morphologies[key] = morph
@@ -346,22 +360,45 @@ class SourceResolver(object):
loader.set_defaults(morph)
return morph
- def _check_version_file(self,definitions_repo,
+ def _parse_version_file(self, version_file): # pragma : no cover
+ '''Parse VERSION file and return the version of the format if:
+
+ VERSION is a YAML file
+ and it's a dict
+ and has the key 'version'
+ and the type stored in the 'version' key is an int
+
+ otherwise returns None
+
+ '''
+
+ yaml_obj = yaml.safe_load(version_file)
+
+ return (yaml_obj['version'] if yaml_obj is not None
+ and isinstance(yaml_obj, dict)
+ and 'version' in yaml_obj
+ and isinstance(yaml_obj['version'], int)
+
+ else None)
+
+ def _check_version_file(self, definitions_repo,
definitions_absref): # pragma: no cover
- version_file = self._get_file_contents(
- definitions_repo, definitions_absref, 'VERSION')
+ version_file = self._get_file_contents(definitions_repo,
+ definitions_absref, 'VERSION')
- if version_file is None:
- return
+ if version_file == None:
+ return 0 # Assume version 0 if no version file
- try:
- version = yaml.safe_load(version_file)['version']
- except (yaml.error.YAMLError, KeyError, TypeError):
- version = 0
+ version = self._parse_version_file(version_file)
- if version in not_supported_versions:
+ if version == None:
+ raise InvalidVersionFileError()
+
+ if version not in supported_versions:
raise UnknownVersionError(version)
+ return version
+
def _process_definitions_with_children(self, system_filenames,
definitions_repo,
definitions_ref,
@@ -371,7 +408,8 @@ class SourceResolver(object):
definitions_queue = collections.deque(system_filenames)
chunk_queue = set()
- self._check_version_file(definitions_repo, definitions_absref)
+ definitions_version = self._check_version_file(definitions_repo,
+ definitions_absref)
while definitions_queue:
filename = definitions_queue.popleft()
@@ -410,9 +448,27 @@ class SourceResolver(object):
# code path should be removed.
path = morphlib.util.sanitise_morphology_path(
c.get('morph', c['name']))
+
chunk_queue.add((c['repo'], c['ref'], path))
else:
- chunk_queue.add((c['repo'], c['ref'], c['morph']))
+ # Now, does this path actually exist?
+ path = c['morph']
+
+ morphology = self._get_morphology(definitions_repo,
+ definitions_absref,
+ path)
+ if morphology is None:
+ if definitions_version > 1:
+ raise MorphologyReferenceNotFoundError(
+ path, filename)
+ else:
+ self.status(
+ msg="Warning! `%(path)s' referenced in "
+ "`%(stratum)s' does not exist",
+ path=path,
+ stratum=filename)
+
+ chunk_queue.add((c['repo'], c['ref'], path))
return chunk_queue
diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py
index 768ec64..df38a2e 100644
--- a/morphlib/stagingarea.py
+++ b/morphlib/stagingarea.py
@@ -108,52 +108,6 @@ class StagingArea(object):
assert filename.startswith(dirname)
return filename[len(dirname) - 1:] # include leading slash
- def hardlink_all_files(self, srcpath, destpath): # pragma: no cover
- '''Hardlink every file in the path to the staging-area
-
- If an exception is raised, the staging-area is indeterminate.
-
- '''
-
- file_stat = os.lstat(srcpath)
- mode = file_stat.st_mode
-
- if stat.S_ISDIR(mode):
- # Ensure directory exists in destination, then recurse.
- if not os.path.lexists(destpath):
- os.makedirs(destpath)
- dest_stat = os.stat(os.path.realpath(destpath))
- if not stat.S_ISDIR(dest_stat.st_mode):
- raise IOError('Destination not a directory. source has %s'
- ' destination has %s' % (srcpath, destpath))
-
- for entry in os.listdir(srcpath):
- self.hardlink_all_files(os.path.join(srcpath, entry),
- os.path.join(destpath, entry))
- elif stat.S_ISLNK(mode):
- # Copy the symlink.
- if os.path.lexists(destpath):
- os.remove(destpath)
- os.symlink(os.readlink(srcpath), destpath)
-
- elif stat.S_ISREG(mode):
- # Hardlink the file.
- if os.path.lexists(destpath):
- os.remove(destpath)
- os.link(srcpath, destpath)
-
- elif stat.S_ISCHR(mode) or stat.S_ISBLK(mode):
- # Block or character device. Put contents of st_dev in a mknod.
- if os.path.lexists(destpath):
- os.remove(destpath)
- os.mknod(destpath, file_stat.st_mode, file_stat.st_rdev)
- os.chmod(destpath, file_stat.st_mode)
-
- else:
- # Unsupported type.
- raise IOError('Cannot extract %s into staging-area. Unsupported'
- ' type.' % srcpath)
-
def create_devices(self, morphology): # pragma: no cover
'''Creates device nodes if the morphology specifies them'''
perms_mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO
@@ -178,17 +132,13 @@ class StagingArea(object):
os.makedev(dev['major'], dev['minor']))
os.chown(destfile, dev['uid'], dev['gid'])
- def install_artifact(self, artifact, artifact_checkout):
- '''Install a build artifact into the staging area.
-
- We access the artifact via an open file handle. For now, we assume
- the artifact is a tarball.
-
- '''
+ def install_artifact(self, artifact_cache, artifact):
+ '''Install a build artifact into the staging area.'''
if not os.path.exists(self.dirname):
self._mkdir(self.dirname)
- self.hardlink_all_files(artifact_checkout, self.dirname)
+ artifact_cache.get(artifact, directory=self.dirname)
+
self.create_devices(artifact.source.morphology)
def remove(self):
diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py
index ffdf5ea..3d37857 100644
--- a/morphlib/stagingarea_tests.py
+++ b/morphlib/stagingarea_tests.py
@@ -46,6 +46,25 @@ class FakeArtifact(object):
self.source = FakeSource()
+class FakeArtifactCache(object):
+
+ def __init__(self, tempdir):
+ self.tempdir = tempdir
+
+ def create_chunk(self, chunkdir):
+ if not chunkdir:
+ chunkdir = os.path.join(self.tempdir, 'chunk')
+ if not os.path.exists(chunkdir):
+ os.mkdir(chunkdir)
+ with open(os.path.join(chunkdir, 'file.txt'), 'w'):
+ pass
+
+ return chunkdir
+
+ def get(self, artifact, directory=None):
+ return self.create_chunk(directory)
+
+
class FakeApplication(object):
def __init__(self, cachedir, tempdir):
@@ -141,14 +160,14 @@ class StagingAreaTests(unittest.TestCase):
def test_installs_artifact(self):
artifact = FakeArtifact()
- chunkdir = self.create_chunk()
- self.sa.install_artifact(artifact, chunkdir)
+ artifact_cache = FakeArtifactCache(self.tempdir)
+ self.sa.install_artifact(artifact_cache, artifact)
self.assertEqual(self.list_tree(self.staging), ['/', '/file.txt'])
def test_removes_everything(self):
artifact = FakeArtifact()
- chunkdir = self.create_chunk()
- self.sa.install_artifact(artifact, chunkdir)
+ artifact_cache = FakeArtifactCache(self.tempdir)
+ self.sa.install_artifact(artifact_cache, artifact)
self.sa.remove()
self.assertFalse(os.path.exists(self.staging))
diff --git a/morphlib/util.py b/morphlib/util.py
index 8566345..9188098 100644
--- a/morphlib/util.py
+++ b/morphlib/util.py
@@ -19,6 +19,7 @@ import pipes
import re
import subprocess
import textwrap
+import sys
import fs.osfs
@@ -119,7 +120,7 @@ def get_git_resolve_cache_server(settings): # pragma: no cover
return None
-def new_artifact_caches(settings): # pragma: no cover
+def new_artifact_caches(settings, status_cb=None): # pragma: no cover
'''Create new objects for local and remote artifact caches.
This includes creating the directories on disk, if missing.
@@ -132,10 +133,8 @@ def new_artifact_caches(settings): # pragma: no cover
os.mkdir(artifact_cachedir)
mode = settings['ostree-repo-mode']
- import logging
- logging.debug(mode)
- lac = morphlib.ostreeartifactcache.OSTreeArtifactCache(artifact_cachedir,
- mode=mode)
+ lac = morphlib.ostreeartifactcache.OSTreeArtifactCache(
+ artifact_cachedir, mode=mode, status_cb=status_cb)
rac_url = get_artifact_cache_server(settings)
rac = None
@@ -453,6 +452,13 @@ def has_hardware_fp(): # pragma: no cover
output = subprocess.check_output(['readelf', '-A', '/proc/self/exe'])
return 'Tag_ABI_VFP_args: VFP registers' in output
+def determine_endianness(): # pragma: no cover
+ '''
+ This function returns whether the host is running
+ in big or little endian. This is needed for MIPS.
+ '''
+
+ return sys.byteorder
def get_host_architecture(): # pragma: no cover
'''Get the canonical Morph name for the host's architecture.'''
@@ -470,7 +476,9 @@ def get_host_architecture(): # pragma: no cover
'armv8l': 'armv8l',
'armv8b': 'armv8b',
'aarch64': 'armv8l64',
- 'aarch64b': 'armv8b64',
+ 'aarch64_be': 'armv8b64',
+ 'mips': 'mips32',
+ 'mips64': 'mips64',
'ppc64': 'ppc64'
}
@@ -479,6 +487,11 @@ def get_host_architecture(): # pragma: no cover
if machine == 'armv7l' and has_hardware_fp():
return 'armv7lhf'
+ elif machine in ('mips', 'mips64'):
+ if determine_endianness() == 'big':
+ return table[machine]+'b'
+ else:
+ return table[machine]+'l'
return table[machine]
@@ -647,3 +660,35 @@ def error_message_for_containerised_commandline(
'Containerisation settings: %s\n' \
'Error output:\n%s' \
% (argv_string, container_kwargs, err)
+
+
+def write_from_dict(filepath, d, validate=lambda x, y: True): #pragma: no cover
+ '''Takes a dictionary and appends the contents to a file
+
+ An optional validation callback can be passed to perform validation on
+ each value in the dictionary.
+
+ e.g.
+
+ def validation_callback(dictionary_key, dictionary_value):
+ if not dictionary_value.isdigit():
+ raise Exception('value contains non-digit character(s)')
+
+ Any callback supplied to this function should raise an exception
+ if validation fails.
+ '''
+
+ # Sort items asciibetically
+ # the output of the deployment should not depend
+ # on the locale of the machine running the deployment
+ items = sorted(d.iteritems(), key=lambda (k, v): [ord(c) for c in v])
+
+ for (k, v) in items:
+ validate(k, v)
+
+ with open(filepath, 'a') as f:
+ for (_, v) in items:
+ f.write('%s\n' % v)
+
+ os.fchown(f.fileno(), 0, 0)
+ os.fchmod(f.fileno(), 0644)
diff --git a/morphlib/writeexts.py b/morphlib/writeexts.py
index 129b2bc..aa185a2 100644
--- a/morphlib/writeexts.py
+++ b/morphlib/writeexts.py
@@ -604,12 +604,16 @@ class WriteExtension(cliapp.Application):
def check_ssh_connectivity(self, ssh_host):
try:
- cliapp.ssh_runcmd(ssh_host, ['true'])
+ output = cliapp.ssh_runcmd(ssh_host, ['echo', 'test'])
except cliapp.AppException as e:
logging.error("Error checking SSH connectivity: %s", str(e))
raise cliapp.AppException(
'Unable to SSH to %s: %s' % (ssh_host, e))
+ if output.strip() != 'test':
+ raise cliapp.AppException(
+ 'Unexpected output from remote machine: %s' % output.strip())
+
def is_device(self, location):
try:
st = os.stat(location)
diff --git a/scripts/check-copyright-year b/scripts/check-copyright-year
index 08bee0a..5a45895 100755
--- a/scripts/check-copyright-year
+++ b/scripts/check-copyright-year
@@ -28,6 +28,7 @@ class CheckCopyrightYear(cliapp.Application):
pat = re.compile(r'^[ #/*]*Copyright\s+(\(C\)\s*)'
r'(?P<years>[0-9, -]+)')
+ ignore = ['COPYING']
def add_settings(self):
self.settings.boolean(['verbose', 'v'], 'be more verbose')
@@ -54,6 +55,9 @@ class CheckCopyrightYear(cliapp.Application):
return filenames
def process_input_line(self, filename, line):
+ if filename in self.ignore:
+ return
+
m = self.pat.match(line)
if not m:
return
diff --git a/tests.build/build-chunk-writes-log.script b/tests.build/build-chunk-writes-log.script
new file mode 100755
index 0000000..5f25757
--- /dev/null
+++ b/tests.build/build-chunk-writes-log.script
@@ -0,0 +1,35 @@
+#!/bin/sh
+#
+# Copyright (C) 2011-2013,2015 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+## Build log should be saved when a chunk is built.
+
+set -eu
+
+"$SRCDIR/scripts/test-morph" build-morphology \
+ test:morphs-repo master hello-system
+
+refsdir="$DATADIR/cache/artifacts/repo/refs/heads"
+chunks=$(find "$refsdir" -name '*-misc' | sed -e "s:$refsdir::" -e "s:-misc::")
+found=false
+
+for chunk in $chunks;
+do
+ [ -e "$DATADIR/cache/artifacts/$chunk".build-log ] || continue
+ found=true
+ break
+done
+"$found"
diff --git a/tests.build/build-stratum-with-submodules.script b/tests.build/build-stratum-with-submodules.script
index a2a1ddc..d1daa29 100755
--- a/tests.build/build-stratum-with-submodules.script
+++ b/tests.build/build-stratum-with-submodules.script
@@ -56,7 +56,20 @@ EOF
"$SRCDIR/scripts/run-git-in" "$morphs" commit --quiet -m 'foo'
-# Now build
+# Now build and verify we got a stratum.
"$SRCDIR/scripts/test-morph" build-morphology \
test:morphs-repo master hello-system
+
+cd "$DATADIR"
+"$SRCDIR/scripts/test-morph" init workspace
+cd "$DATADIR/workspace"
+"$SRCDIR/scripts/test-morph" checkout test:morphs-repo master
+cd "$DATADIR/workspace/master/test/morphs-repo"
+
+ref=`"$SRCDIR/scripts/test-morph" query-cache hello-system.morph`
+ostree --repo="$DATADIR/cache/artifacts/repo" checkout --fsync=false \
+ "$ref" "$DATADIR/$ref"
+find $DATADIR/$ref/* | sed "s:^$DATADIR/$ref/::" | LC_ALL=C sort |
+sed '/^\.\/./s:^\./::' | grep -v '^baserock'
+
diff --git a/tests.build/build-stratum-with-submodules.stdout b/tests.build/build-stratum-with-submodules.stdout
new file mode 100644
index 0000000..864f253
--- /dev/null
+++ b/tests.build/build-stratum-with-submodules.stdout
@@ -0,0 +1,2 @@
+etc
+etc/os-release
diff --git a/tests.build/build-system-autotools.script b/tests.build/build-system-autotools.script
index 936fa49..7ecb31b 100755
--- a/tests.build/build-system-autotools.script
+++ b/tests.build/build-system-autotools.script
@@ -46,3 +46,16 @@ git commit --quiet -m "Convert hello to an autotools project"
"$SRCDIR/scripts/test-morph" build-morphology \
test:morphs-repo master hello-system
+
+cd "$DATADIR"
+"$SRCDIR/scripts/test-morph" init workspace
+cd "$DATADIR/workspace"
+"$SRCDIR/scripts/test-morph" checkout test:morphs-repo master
+cd "$DATADIR/workspace/master/test/morphs-repo"
+
+refs=`"$SRCDIR/scripts/test-morph" query-cache hello-system.morph hello`
+for ref in $refs
+do
+ ostree --repo="$DATADIR/cache/artifacts/repo" checkout --fsync=false "$ref" "$DATADIR/$ref"
+ find $DATADIR/$ref/* | sed "s:^$DATADIR/$ref/::"
+done | LC_ALL=C sort -u | sed '/^\.\/./s:^\./::' | grep -Ee '^(bin|etc)'
diff --git a/tests.build/build-system-autotools.stdout b/tests.build/build-system-autotools.stdout
new file mode 100644
index 0000000..6dd6cda
--- /dev/null
+++ b/tests.build/build-system-autotools.stdout
@@ -0,0 +1,3 @@
+bin
+bin/hello
+etc
diff --git a/tests.build/build-system-cmake.script b/tests.build/build-system-cmake.script
index b848aab..b761a5d 100755
--- a/tests.build/build-system-cmake.script
+++ b/tests.build/build-system-cmake.script
@@ -48,3 +48,17 @@ git commit --quiet -m "Convert hello to a cmake project"
"$SRCDIR/scripts/test-morph" build-morphology \
test:morphs-repo master hello-system
+
+cd "$DATADIR"
+"$SRCDIR/scripts/test-morph" init workspace
+cd "$DATADIR/workspace"
+"$SRCDIR/scripts/test-morph" checkout test:morphs-repo master
+cd "$DATADIR/workspace/master/test/morphs-repo"
+
+refs=`"$SRCDIR/scripts/test-morph" query-cache hello-system.morph hello`
+for ref in $refs
+do
+ ostree --repo="$DATADIR/cache/artifacts/repo" checkout --fsync=false \
+ "$ref" "$DATADIR/$ref"
+ find $DATADIR/$ref/* | sed "s:^$DATADIR/$ref/::"
+done | LC_ALL=C sort -u | sed '/^\.\/./s:^\./::' | grep -Ee '^(usr/)?(bin|etc)'
diff --git a/tests.build/build-system-cmake.stdout b/tests.build/build-system-cmake.stdout
new file mode 100644
index 0000000..861fd1f
--- /dev/null
+++ b/tests.build/build-system-cmake.stdout
@@ -0,0 +1,2 @@
+usr/bin
+usr/bin/hello
diff --git a/tests.build/build-system-cpan.script b/tests.build/build-system-cpan.script
index b686de3..e6bd579 100755
--- a/tests.build/build-system-cpan.script
+++ b/tests.build/build-system-cpan.script
@@ -70,3 +70,17 @@ git commit -q -m "Set custom install prefix for hello"
"$SRCDIR/scripts/test-morph" build-morphology \
test:morphs-repo master hello-system
+
+cd "$DATADIR"
+"$SRCDIR/scripts/test-morph" init workspace
+cd "$DATADIR/workspace"
+"$SRCDIR/scripts/test-morph" checkout test:morphs-repo master
+cd "$DATADIR/workspace/master/test/morphs-repo"
+
+refs=`"$SRCDIR/scripts/test-morph" query-cache hello-system.morph hello`
+for ref in $refs
+do
+ ostree --repo="$DATADIR/cache/artifacts/repo" checkout --fsync=false \
+ "$ref" "$DATADIR/$ref"
+ find $DATADIR/$ref/* | sed "s:^$DATADIR/$ref/::"
+done | LC_ALL=C sort | sed '/^\.\/./s:^\./::' | grep -F 'bin/hello'
diff --git a/tests.build/build-system-cpan.stdout b/tests.build/build-system-cpan.stdout
new file mode 100644
index 0000000..180e949
--- /dev/null
+++ b/tests.build/build-system-cpan.stdout
@@ -0,0 +1 @@
+bin/hello
diff --git a/tests.build/build-system-python-distutils.script b/tests.build/build-system-python-distutils.script
index d821031..4441865 100755
--- a/tests.build/build-system-python-distutils.script
+++ b/tests.build/build-system-python-distutils.script
@@ -68,3 +68,22 @@ git commit -q -m "Set custom install prefix for hello"
"$SRCDIR/scripts/test-morph" build-morphology \
test:morphs-repo master hello-system
+
+cd "$DATADIR"
+"$SRCDIR/scripts/test-morph" init workspace
+cd "$DATADIR/workspace"
+"$SRCDIR/scripts/test-morph" checkout test:morphs-repo master
+cd "$DATADIR/workspace/master/test/morphs-repo"
+
+refs=`"$SRCDIR/scripts/test-morph" query-cache hello-system.morph hello`
+for ref in $refs
+do
+ ostree --repo="$DATADIR/cache/artifacts/repo" checkout --fsync=false \
+ "$ref" "$DATADIR/$ref"
+ find $DATADIR/$ref/* | sed "s:^$DATADIR/$ref/::"
+done | LC_ALL=C sort -u | sed '/^\.\/./s:^\./::' | grep -Ee '^(bin|lib)' |
+sed -e 's:^local/::' \
+ -e 's:lib/python2.[6-9]:lib/python2.x:' \
+ -e 's:/hello-0\.0\.0[^/]*\.egg-info$:/hello.egg-info:' \
+ -e 's:[^/]*-packages:packages:' \
+ -e '/^$/d'
diff --git a/tests.build/build-system-python-distutils.stdout b/tests.build/build-system-python-distutils.stdout
new file mode 100644
index 0000000..a2ceb5a
--- /dev/null
+++ b/tests.build/build-system-python-distutils.stdout
@@ -0,0 +1,6 @@
+bin
+bin/hello
+lib
+lib/python2.x
+lib/python2.x/packages
+lib/python2.x/packages/hello.egg-info
diff --git a/tests.build/build-system-qmake.script b/tests.build/build-system-qmake.script
index b477de4..d430fba 100755
--- a/tests.build/build-system-qmake.script
+++ b/tests.build/build-system-qmake.script
@@ -22,6 +22,7 @@ set -eu
if ! command -v qmake > /dev/null ; then
# There is no qmake, so skip this test.
+ cat "$SRCDIR/tests.build/build-system-qmake.stdout"
exit 0
fi
@@ -55,3 +56,10 @@ git commit --quiet -m "Convert hello to an qmake project"
"$SRCDIR/scripts/test-morph" build-morphology \
test:morphs-repo master hello-system
+
+for chunk in "$DATADIR/cache/artifacts/"*.chunk.*
+do
+ echo "$chunk:" | sed 's/[^.]*//'
+ tar -tf "$chunk" | LC_ALL=C sort | sed '/^\.\/./s:^\./::'
+ echo
+done
diff --git a/tests.build/build-system-qmake.stdout b/tests.build/build-system-qmake.stdout
new file mode 100644
index 0000000..ccf80a8
--- /dev/null
+++ b/tests.build/build-system-qmake.stdout
@@ -0,0 +1,8 @@
+.chunk.hello:
+./
+baserock/
+baserock/hello.meta
+usr/
+usr/bin/
+usr/bin/hello
+
diff --git a/tests.build/build-system.script b/tests.build/build-system.script
new file mode 100755
index 0000000..d3e338c
--- /dev/null
+++ b/tests.build/build-system.script
@@ -0,0 +1,35 @@
+#!/bin/sh
+#
+# Copyright (C) 2011-2015 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+## Test building a simple system.
+
+set -eu
+
+"$SRCDIR/scripts/test-morph" build-morphology \
+ test:morphs-repo master hello-system
+
+cd "$DATADIR"
+"$SRCDIR/scripts/test-morph" init workspace
+cd "$DATADIR/workspace"
+"$SRCDIR/scripts/test-morph" checkout test:morphs-repo master
+cd "$DATADIR/workspace/master/test/morphs-repo"
+
+ref=`"$SRCDIR/scripts/test-morph" query-cache hello-system.morph`
+ostree --repo="$DATADIR/cache/artifacts/repo" checkout --fsync=false \
+ "$ref" "$DATADIR/$ref"
+find $DATADIR/$ref/* | sed "s:^$DATADIR/$ref/::" | LC_ALL=C sort |
+sed '/^\.\/./s:^\./::' | grep -v '^baserock'
diff --git a/tests.build/build-system.stdout b/tests.build/build-system.stdout
new file mode 100644
index 0000000..864f253
--- /dev/null
+++ b/tests.build/build-system.stdout
@@ -0,0 +1,2 @@
+etc
+etc/os-release
diff --git a/tests.build/cross-bootstrap.script b/tests.build/cross-bootstrap.script
index 6bab165..eb9ade3 100755
--- a/tests.build/cross-bootstrap.script
+++ b/tests.build/cross-bootstrap.script
@@ -20,11 +20,11 @@
set -eu
-"$SRCDIR/tests.build/setup-build-essential"
-
# cross-bootstrap needs rewriting for OSTree
exit 0
+"$SRCDIR/tests.build/setup-build-essential"
+
"$SRCDIR/scripts/test-morph" cross-bootstrap \
$("$SRCDIR/scripts/test-morph" print-architecture) \
test:morphs-repo master hello-system
diff --git a/tests.build/morphless-chunks.script b/tests.build/morphless-chunks.script
index b46fa63..f0eb151 100755
--- a/tests.build/morphless-chunks.script
+++ b/tests.build/morphless-chunks.script
@@ -40,3 +40,16 @@ git commit -q -m "Convert hello into an autodetectable chunk"
"$SRCDIR/scripts/test-morph" build-morphology \
test:morphs-repo master hello-system
+
+cd "$DATADIR"
+"$SRCDIR/scripts/test-morph" init workspace
+cd "$DATADIR/workspace"
+"$SRCDIR/scripts/test-morph" checkout test:morphs-repo master
+cd "$DATADIR/workspace/master/test/morphs-repo"
+
+refs=`"$SRCDIR/scripts/test-morph" query-cache hello-system.morph hello`
+for ref in $refs
+do
+ ostree --repo="$DATADIR/cache/artifacts/repo" checkout --fsync=false \
+ "$ref" "$DATADIR/$ref"
+done | cat >/dev/null
diff --git a/tests.build/morphless-chunks.stdout b/tests.build/morphless-chunks.stdout
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests.build/morphless-chunks.stdout
diff --git a/tests.build/only-build-systems.stderr b/tests.build/only-build-systems.stderr
index ba7339d..ac24ab2 100644
--- a/tests.build/only-build-systems.stderr
+++ b/tests.build/only-build-systems.stderr
@@ -1,2 +1,2 @@
-ERROR: Building a stratum directly is not supported
-ERROR: Building a chunk directly is not supported
+ERROR: In order to build this stratum directly, please give the filename of the system which contains it, and the name of the stratum. See `morph build --help` for more information.
+ERROR: In order to build this chunk directly, please give the filename of the system which contains it, and the name of the chunk. See `morph build --help` for more information.
diff --git a/tests.build/prefix.script b/tests.build/prefix.script
index 75c9120..a87671c 100755
--- a/tests.build/prefix.script
+++ b/tests.build/prefix.script
@@ -65,3 +65,17 @@ git commit -q -m "Update stratum"
"$SRCDIR/scripts/test-morph" build-morphology \
test:morphs-repo master hello-system
+
+cd "$DATADIR"
+"$SRCDIR/scripts/test-morph" init workspace
+cd "$DATADIR/workspace"
+"$SRCDIR/scripts/test-morph" checkout test:morphs-repo master
+
+cd "$DATADIR/workspace/master/test/morphs-repo"
+test_morph="$SRCDIR/scripts/test-morph"
+first_chunk=$("$test_morph" query-cache hello-system.morph xyzzy | head -n1 |
+ cut -c -64)
+second_chunk=$("$test_morph" query-cache hello-system.morph plugh | head -n1 |
+ cut -c -64)
+cd "$DATADIR/cache/artifacts"
+cat $first_chunk.build-log $second_chunk.build-log
diff --git a/tests.build/prefix.stdout b/tests.build/prefix.stdout
new file mode 100644
index 0000000..80c18fa
--- /dev/null
+++ b/tests.build/prefix.stdout
@@ -0,0 +1,8 @@
+# configure
+# # echo First chunk: prefix $PREFIX
+First chunk: prefix /plover
+# configure
+# # echo Second chunk: prefix $PREFIX
+Second chunk: prefix /usr
+# # echo Path: $(echo $PATH | grep -o '/plover')
+Path: /plover
diff --git a/tests.build/rebuild-cached-stratum.script b/tests.build/rebuild-cached-stratum.script
index bdbe193..dacd441 100755
--- a/tests.build/rebuild-cached-stratum.script
+++ b/tests.build/rebuild-cached-stratum.script
@@ -40,6 +40,9 @@ cache="$DATADIR/cache/artifacts"
# Build the first time.
"$SRCDIR/scripts/test-morph" build-morphology \
test:morphs-repo rebuild-cached-stratum hello-system
+echo "first build:"
+(cd "$cache" && ls *hello-stratum-* | sed 's/^[^.]*\./ /' |
+ LC_ALL=C sort -u)
# Change the chunk.
(cd "$DATADIR/chunk-repo" &&
@@ -49,3 +52,7 @@ cache="$DATADIR/cache/artifacts"
# Rebuild.
"$SRCDIR/scripts/test-morph" build-morphology \
test:morphs-repo rebuild-cached-stratum hello-system
+echo "second build:"
+(cd "$cache" && ls *hello-stratum-* | sed 's/^[^.]*\./ /' |
+ LC_ALL=C sort -u)
+
diff --git a/tests.build/rebuild-cached-stratum.stdout b/tests.build/rebuild-cached-stratum.stdout
new file mode 100644
index 0000000..7a61bc5
--- /dev/null
+++ b/tests.build/rebuild-cached-stratum.stdout
@@ -0,0 +1,10 @@
+first build:
+ stratum.hello-stratum-devel
+ stratum.hello-stratum-devel.meta
+ stratum.hello-stratum-runtime
+ stratum.hello-stratum-runtime.meta
+second build:
+ stratum.hello-stratum-devel
+ stratum.hello-stratum-devel.meta
+ stratum.hello-stratum-runtime
+ stratum.hello-stratum-runtime.meta
diff --git a/without-test-modules b/without-test-modules
index 2e1b8c5..d7173b6 100644
--- a/without-test-modules
+++ b/without-test-modules
@@ -26,11 +26,13 @@ morphlib/plugins/__init__.py
morphlib/writeexts.py
morphlib/plugins/list_artifacts_plugin.py
morphlib/plugins/trovectl_plugin.py
+morphlib/plugins/get_chunk_details_plugin.py
morphlib/plugins/gc_plugin.py
morphlib/plugins/print_architecture_plugin.py
morphlib/plugins/add_binary_plugin.py
morphlib/plugins/push_pull_plugin.py
morphlib/plugins/distbuild_plugin.py
+morphlib/plugins/certify_plugin.py
distbuild/__init__.py
distbuild/build_controller.py
distbuild/connection_machine.py
@@ -54,3 +56,4 @@ distbuild/worker_build_scheduler.py
morphlib/buildbranch.py
morphlib/ostree.py
morphlib/ostreeartifactcache.py
+morphlib/plugins/ostree_artifacts_plugin.py
diff --git a/yarns/building.yarn b/yarns/building.yarn
index b5e46b7..8a98e5d 100644
--- a/yarns/building.yarn
+++ b/yarns/building.yarn
@@ -112,3 +112,14 @@ Empty strata don't build
AND the user attempts to build the system systems/empty-stratum-system.morph in branch empty-stratum
THEN morph failed
FINALLY the git server is shut down
+
+Partial building
+----------------
+
+ SCENARIO partial building
+ GIVEN a workspace
+ AND a git server
+ WHEN the user checks out the system branch called master
+ AND the user attempts to build build-essential from the system systems/test-system.morph in branch master
+ THEN morph succeeded
+ FINALLY the git server is shut down
diff --git a/yarns/deployment.yarn b/yarns/deployment.yarn
index 6ec8c0a..85bb2c9 100644
--- a/yarns/deployment.yarn
+++ b/yarns/deployment.yarn
@@ -301,6 +301,28 @@ deployment.
THEN morph failed
FINALLY the git server is shut down
+Deploying only part of a system
+-------------------------------
+
+It is possible to only deploy one or more strata or chunks from a system
+when deploying to a tarball or sysroot.
+
+ SCENARIO partially deploying a system
+ GIVEN a workspace
+ AND a git server
+ WHEN the user checks out the system branch called master
+ AND the user builds the system systems/test-system.morph in branch master
+ AND the user attempts to partially deploy tar from cluster clusters/partial-test-cluster.morph in branch master
+ THEN morph succeeded
+ WHEN the user attempts to partially deploy sysroot from cluster clusters/partial-test-cluster.morph in branch master
+ THEN morph succeeded
+
+However, it is not possible to do this when the deployment type is
+something other than tarball or sysroot.
+
+ WHEN the user attempts to partially deploy rawdisk from cluster clusters/partial-test-cluster.morph in branch master
+ THEN morph failed
+
Deploying branch-from-image produced systems
============================================
diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn
index fb110ff..6965ebe 100644
--- a/yarns/implementations.yarn
+++ b/yarns/implementations.yarn
@@ -332,6 +332,29 @@ another to hold a chunk.
- copy files
EOF
+ install -m644 -D /dev/stdin << EOF "clusters/partial-test-cluster.morph"
+ name: partial-test-cluster
+ kind: cluster
+ systems:
+ - morph: systems/test-system.morph
+ deploy:
+ tar:
+ type: tar
+ location: test.tar
+ partial-deploy-components:
+ - strata/build-essential.morph
+ sysroot:
+ type: sysroot
+ location: test.sysroot
+ partial-deploy-components:
+ - strata/build-essential.morph
+ rawdisk:
+ type: rawdisk
+ location: test.img
+ partial-deploy-components:
+ - strata/build-essential.morph
+ EOF
+
git add .
git commit -m Initial.
git tag -a "test-tag" -m "Tagging test-tag"
@@ -440,7 +463,6 @@ You need an architecture to build a system, we don't default to the host archite
run_in "$DATADIR/gits/morphs" git add "$MATCH_1"
run_in "$DATADIR/gits/morphs" git commit -m "Added $MATCH_1."
-
Implementation sections for system branch operations
----------------------------------------------------
@@ -737,6 +759,12 @@ Implementation sections for building
if [ "$MATCH_1" != "attempts to " ]; then run_morph "$@"
else attempt_morph "$@"; fi
+ IMPLEMENTS WHEN the user (attempts to )?((dist)?build)s? (\S+) from the system (\S+) in branch (\S+)
+ cd "$DATADIR/workspace/$MATCH_6/test/morphs"
+ set "$MATCH_2" "$MATCH_5" "$MATCH_4"
+ if [ "$MATCH_1" != "attempts to " ]; then run_morph "$@"
+ else attempt_morph "$@"; fi
+
Implementation sections for cross-bootstraping
==============================================
@@ -782,6 +810,15 @@ them, so they can be added to the end of the implements section.
if [ "$MATCH_1" = "deploys" ]; then run_morph "$@"
else attempt_morph "$@"; fi
+ IMPLEMENTS WHEN the user (attempts to partially deploy|partially deploys) (.*) from cluster (\S+) in branch (\S+)
+ cd "$DATADIR/workspace/$MATCH_4/test/morphs"
+ set -- deploy "$MATCH_3"
+ systems=$(echo "$MATCH_2" | sed -e 's/, /\n/g' -e 's/ and /\n/g')
+ echo "partial=True" >> "$DATADIR/morph.conf"
+ set -- "$@" $systems
+ if [ "$MATCH_1" = "deploys" ]; then run_morph "$@"
+ else attempt_morph "$@"; fi
+
IMPLEMENTS WHEN the user (attempts to upgrade|upgrades) the (system|cluster) (\S+) in branch (\S+)( with options (.*))?
cd "$DATADIR/workspace/$MATCH_4/test/morphs"
set -- upgrade "$MATCH_3"