Merge tag 'dm-pull-29jul19' of https://gitlab.denx.de/u-boot/custodia…

…ns/u-boot-dm

binman support for replacing files

Makefile

@@ -1189,9 +1189,10 @@ u-boot.ldr:	u-boot
 # ---------------------------------------------------------------------------
 # Use 'make BINMAN_DEBUG=1' to enable debugging
 quiet_cmd_binman = BINMAN  $@
-cmd_binman = $(srctree)/tools/binman/binman build -u -d u-boot.dtb -O . -m \
+cmd_binman = $(srctree)/tools/binman/binman $(if $(BINMAN_DEBUG),-D) \
+                build -u -d u-boot.dtb -O . -m \
 		-I . -I $(srctree) -I $(srctree)/board/$(BOARDDIR) \
-		$(if $(BINMAN_DEBUG),-D) $(BINMAN_$(@F))
+		$(BINMAN_$(@F))
 
 OBJCOPYFLAGS_u-boot.ldr.hex := -I binary -O ihex
 

scripts/dtc/libfdt/fdt_sw.c

@@ -114,7 +114,7 @@ int fdt_resize(void *fdt, void *buf, int bufsize)
 
 	FDT_SW_CHECK_HEADER(fdt);
 
-	headsize = fdt_off_dt_struct(fdt);
+	headsize = fdt_off_dt_struct(fdt) + fdt_size_dt_struct(fdt);
 	tailsize = fdt_size_dt_strings(fdt);
 
 	if ((headsize + tailsize) > bufsize)

tools/binman/README

@@ -481,6 +481,29 @@ name-prefix:
 	distinguish binaries with otherwise identical names.
 
 
+Image Properties
+----------------
+
+Image nodes act like sections but also have a few extra properties:
+
+filename:
+	Output filename for the image. This defaults to image.bin (or in the
+	case of multiple images <nodename>.bin where <nodename> is the name of
+	the image node.
+
+allow-repack:
+	Create an image that can be repacked. With this option it is possible
+	to change anything in the image after it is created, including updating
+	the position and size of image components. By default this is not
+	permitted since it is not possibly to know whether this might violate a
+	constraint in the image description. For example, if a section has to
+	increase in size to hold a larger binary, that might cause the section
+	to fall out of its allow region (e.g. read-only portion of flash).
+
+	Adding this property causes the original offset and size values in the
+	image description to be stored in the FDT and fdtmap.
+
+
 Entry Documentation
 -------------------
 
@@ -557,6 +580,35 @@ or just a selection:
     $ binman extract -i image.bin "*u-boot*" -O outdir
 
 
+Replacing files in an image
+---------------------------
+
+You can replace files in an existing firmware image created by binman, provided
+that there is an 'fdtmap' entry in the image. For example:
+
+    $ binman replace -i image.bin section/cbfs/u-boot
+
+which will write the contents of the file 'u-boot' from the current directory
+to the that entry, compressing if necessary. If the entry size changes, you must
+add the 'allow-repack' property to the original image before generating it (see
+above), otherwise you will get an error.
+
+You can also use a particular file, in this case u-boot.bin:
+
+    $ binman replace -i image.bin section/cbfs/u-boot -f u-boot.bin
+
+It is possible to replace all files from a source directory which uses the same
+hierarchy as the entries:
+
+    $ binman replace -i image.bin -I indir
+
+Files that are missing will generate a warning.
+
+You can also replace just a selection of entries:
+
+    $ binman replace -i image.bin "*u-boot*" -I indir
+
+
 Logging
 -------
 
@@ -644,22 +696,35 @@ large enough to hold all the entries.
 7. CheckEntries() - checks that the entries do not overlap, nor extend
 outside the image.
 
-8. SetCalculatedProperties() - update any calculated properties in the device
+8. SetImagePos() - sets the image position of every entry. This is the absolute
+position 'image-pos', as opposed to 'offset' which is relative to the containing
+section. This must be done after all offsets are known, which is why it is quite
+late in the ordering.
+
+9. SetCalculatedProperties() - update any calculated properties in the device
 tree. This sets the correct 'offset' and 'size' vaues, for example.
 
-9. ProcessEntryContents() - this calls Entry.ProcessContents() on each entry.
+10. ProcessEntryContents() - this calls Entry.ProcessContents() on each entry.
 The default implementatoin does nothing. This can be overriden to adjust the
 contents of an entry in some way. For example, it would be possible to create
 an entry containing a hash of the contents of some other entries. At this
 stage the offset and size of entries should not be adjusted unless absolutely
 necessary, since it requires a repack (going back to PackEntries()).
 
-10. WriteSymbols() - write the value of symbols into the U-Boot SPL binary.
+11. ResetForPack() - if the ProcessEntryContents() step failed, in that an entry
+has changed its size, then there is no alternative but to go back to step 5 and
+try again, repacking the entries with the updated size. ResetForPack() removes
+the fixed offset/size values added by binman, so that the packing can start from
+scratch.
+
+12. WriteSymbols() - write the value of symbols into the U-Boot SPL binary.
 See 'Access to binman entry offsets at run time' below for a description of
 what happens in this stage.
 
-11. BuildImage() - builds the image and writes it to a file. This is the final
-step.
+13. BuildImage() - builds the image and writes it to a file
+
+14. WriteMap() - writes a text file containing a map of the image. This is the
+final step.
 
 
 Automatic .dtsi inclusion
@@ -909,10 +974,10 @@ Some ideas:
 - Allow easy building of images by specifying just the board name
 - Support building an image for a board (-b) more completely, with a
   configurable build directory
-- Support updating binaries in an image (with no size change / repacking)
-- Support updating binaries in an image (with repacking)
 - Support adding FITs to an image
 - Support for ARM Trusted Firmware (ATF)
+- Detect invalid properties in nodes
+- Sort the fdtmap by offset
 
 --
 Simon Glass <[email protected]>

tools/binman/README.entries

@@ -230,7 +230,9 @@ Properties / Entry arguments:
     None
 
 An FDT map is just a header followed by an FDT containing a list of all the
-entries in the image.
+entries in the image. The root node corresponds to the image node in the
+original FDT, and an image-name property indicates the image name in that
+original tree.
 
 The header is the string _FDTMAP_ followed by 8 unused bytes.
 
@@ -244,6 +246,7 @@ FDT with the position of each entry.
 Example output for a simple image with U-Boot and an FDT map:
 
 / {
+    image-name = "binman";
     size = <0x00000112>;
     image-pos = <0x00000000>;
     offset = <0x00000000>;
@@ -259,6 +262,9 @@ Example output for a simple image with U-Boot and an FDT map:
     };
 };
 
+If allow-repack is used then 'orig-offset' and 'orig-size' properties are
+added as necessary. See the binman README.
+
 
 
 Entry: files: Entry containing a set of files
@@ -308,7 +314,8 @@ see www.flashrom.org/Flashrom for more information.
 
 When used, this entry will be populated with an FMAP which reflects the
 entries in the current image. Note that any hierarchy is squashed, since
-FMAP does not support this.
+FMAP does not support this. Also, CBFS entries appear as a single entry -
+the sub-entries are ignored.
 
 
 

tools/binman/cbfs_util.py

@@ -208,6 +208,7 @@ class CbfsFile(object):
         cbfs_offset: Offset of file data in bytes from start of CBFS, or None to
             place this file anyway
         data: Contents of file, uncompressed
+        orig_data: Original data added to the file, possibly compressed
         data_len: Length of (possibly compressed) data in bytes
         ftype: File type (TYPE_...)
         compression: Compression type (COMPRESS_...)
@@ -226,6 +227,7 @@ def __init__(self, name, ftype, data, cbfs_offset, compress=COMPRESS_NONE):
         self.offset = None
         self.cbfs_offset = cbfs_offset
         self.data = data
+        self.orig_data = data
         self.ftype = ftype
         self.compress = compress
         self.memlen = None
@@ -240,9 +242,9 @@ def decompress(self):
         """Handle decompressing data if necessary"""
         indata = self.data
         if self.compress == COMPRESS_LZ4:
-            data = tools.Decompress(indata, 'lz4')
+            data = tools.Decompress(indata, 'lz4', with_header=False)
         elif self.compress == COMPRESS_LZMA:
-            data = tools.Decompress(indata, 'lzma')
+            data = tools.Decompress(indata, 'lzma', with_header=False)
         else:
             data = indata
         self.memlen = len(data)
@@ -361,9 +363,9 @@ def get_data_and_offset(self, offset=None, pad_byte=None):
         elif self.ftype == TYPE_RAW:
             orig_data = data
             if self.compress == COMPRESS_LZ4:
-                data = tools.Compress(orig_data, 'lz4')
+                data = tools.Compress(orig_data, 'lz4', with_header=False)
             elif self.compress == COMPRESS_LZMA:
-                data = tools.Compress(orig_data, 'lzma')
+                data = tools.Compress(orig_data, 'lzma', with_header=False)
             self.memlen = len(orig_data)
             self.data_len = len(data)
             attr = struct.pack(ATTR_COMPRESSION_FORMAT,
@@ -715,7 +717,7 @@ def _read_next_file(self, fd):
         file_pos = fd.tell()
         data = fd.read(FILE_HEADER_LEN)
         if len(data) < FILE_HEADER_LEN:
-            print('File header at %x ran out of data' % file_pos)
+            print('File header at %#x ran out of data' % file_pos)
             return False
         magic, size, ftype, attr, offset = struct.unpack(FILE_HEADER_FORMAT,
                                                          data)
@@ -724,7 +726,7 @@ def _read_next_file(self, fd):
         pos = fd.tell()
         name = self._read_string(fd)
         if name is None:
-            print('String at %x ran out of data' % pos)
+            print('String at %#x ran out of data' % pos)
             return False
 
         if DEBUG:

tools/binman/cbfs_util_test.py

@@ -372,7 +372,7 @@ def test_cbfs_bad_file_header(self):
             with io.BytesIO(newdata) as fd:
                 fd.seek(pos)
                 self.assertEqual(False, cbr._read_next_file(fd))
-        self.assertIn('File header at 0 ran out of data', stdout.getvalue())
+        self.assertIn('File header at 0x0 ran out of data', stdout.getvalue())
 
     def test_cbfs_bad_file_string(self):
         """Check handling of an incomplete filename string"""
@@ -394,7 +394,7 @@ def test_cbfs_bad_file_string(self):
             with io.BytesIO(newdata) as fd:
                 fd.seek(pos)
                 self.assertEqual(False, cbr._read_next_file(fd))
-        self.assertIn('String at %x ran out of data' %
+        self.assertIn('String at %#x ran out of data' %
                       cbfs_util.FILE_HEADER_LEN, stdout.getvalue())
 
     def test_cbfs_debug(self):

tools/binman/cmdline.py

@@ -84,6 +84,23 @@ def ParseArgs(argv):
     extract_parser.add_argument('-U', '--uncompressed', action='store_true',
         help='Output raw uncompressed data for compressed entries')
 
+    replace_parser = subparsers.add_parser('replace',
+                                           help='Replace entries in an image')
+    replace_parser.add_argument('-C', '--compressed', action='store_true',
+        help='Input data is already compressed if needed for the entry')
+    replace_parser.add_argument('-i', '--image', type=str, required=True,
+                                help='Image filename to extract')
+    replace_parser.add_argument('-f', '--filename', type=str,
+                                help='Input filename to read from')
+    replace_parser.add_argument('-F', '--fix-size', action='store_true',
+        help="Don't allow entries to be resized")
+    replace_parser.add_argument('-I', '--indir', type=str, default='',
+        help='Path to directory to use for input files')
+    replace_parser.add_argument('-m', '--map', action='store_true',
+        default=False, help='Output a map file for the updated image')
+    replace_parser.add_argument('paths', type=str, nargs='*',
+                                help='Paths within file to extract (wildcard)')
+
     test_parser = subparsers.add_parser('test', help='Run tests')
     test_parser.add_argument('-P', '--processes', type=int,
         help='set number of processes to use for running tests')