Skip to content

Commit

Permalink
decode ggla
Browse files Browse the repository at this point in the history
  • Loading branch information
mxyng committed Mar 8, 2024
1 parent 18979ad commit 76bdebb
Show file tree
Hide file tree
Showing 5 changed files with 171 additions and 33 deletions.
2 changes: 1 addition & 1 deletion convert/convert.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ func ReadSafeTensors(fn string, offset uint64) ([]llm.Tensor, uint64, error) {
Name: ggufName,
Kind: kind,
Offset: offset,
Shape: shape,
Shape: shape[:],
FileName: fn,
OffsetPadding: 8 + jsonSize,
FileOffsets: []uint64{uint64(data.Offsets[0]), uint64(data.Offsets[1])},
Expand Down
152 changes: 152 additions & 0 deletions llm/ggla.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
package llm

import (
"encoding/binary"
"errors"
"io"
"slices"
)

type ContainerGGLA struct {
version uint32
}

func (c *ContainerGGLA) Name() string {
return "ggla"
}

func (c *ContainerGGLA) Decode(rso *readSeekOffset) (model, error) {
binary.Read(rso, binary.LittleEndian, &c.version)

switch c.version {
case 1:
default:
return nil, errors.New("invalid version")
}

model := newModelGGLA(c)
err := model.decode(rso)
return model, err
}

type ModelGGLA struct {
*ContainerGGLA

kv KV
tensors []Tensor
}

func newModelGGLA(container *ContainerGGLA) *ModelGGLA {
return &ModelGGLA{
ContainerGGLA: container,
kv: make(KV),
}
}

func (m *ModelGGLA) decode(rso *readSeekOffset) error {
var r uint32
if err := binary.Read(rso, binary.LittleEndian, &r); err != nil {
return err
}
m.kv["r"] = r

var alpha uint32
if err := binary.Read(rso, binary.LittleEndian, &alpha); err != nil {
return err
}
m.kv["alpha"] = alpha

for {
var dims uint32
if err := binary.Read(rso, binary.LittleEndian, &dims); err != nil {
return err
}

var namesize uint32
if err := binary.Read(rso, binary.LittleEndian, &namesize); err != nil {
return err
}

var t Tensor
if err := binary.Read(rso, binary.LittleEndian, &t.Kind); err != nil {
return err
}

t.Shape = make([]uint64, dims)
for i := 0; uint32(i) < dims; i++ {
var shape32 uint32
if err := binary.Read(rso, binary.LittleEndian, &shape32); err != nil {
return err
}

t.Shape[i] = uint64(shape32)
}

// ggla tensor shape is reversed
// ref: https://github.com/ggerganov/llama.cpp/blob/29ae62d2ae163e2b68aa0ad3bf2ab4636de0c957/convert-lora-to-ggml.py#L44
slices.Reverse(t.Shape)

name := make([]byte, namesize)
if err := binary.Read(rso, binary.LittleEndian, &name); err != nil {
return err
}

t.Name = string(name)

if _, err := rso.Seek((rso.offset+31)&-32, io.SeekStart); err != nil {
return err
}

t.Offset = uint64(rso.offset)

if _, err := rso.Seek(int64(t.Size()), io.SeekCurrent); err != nil {
return err
}

m.tensors = append(m.tensors, t)
}
}

func (m *ModelGGLA) KV() KV {
return m.kv
}

func (m *ModelGGLA) Tensor() []Tensor {
return m.tensors
}

func (*ModelGGLA) ModelFamily() string {
return "ggla"
}

func (*ModelGGLA) ModelType() string {
panic("not implemented")
}

func (*ModelGGLA) FileType() string {
panic("not implemented")
}

func (*ModelGGLA) NumLayers() uint32 {
panic("not implemented")
}

func (*ModelGGLA) NumGQA() uint32 {
panic("not implemented")
}

func (*ModelGGLA) NumEmbed() uint32 {
panic("not implemented")
}

func (*ModelGGLA) NumHead() uint32 {
panic("not implemented")
}

func (*ModelGGLA) NumHeadKv() uint32 {
panic("not implemented")
}

func (*ModelGGLA) NumCtx() uint32 {
panic("not implemented")
}
32 changes: 4 additions & 28 deletions llm/ggml.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,32 +106,6 @@ type container interface {
Decode(*readSeekOffset) (model, error)
}

type containerLORA struct {
version uint32
}

func (c *containerLORA) Name() string {
return "ggla"
}

func (c *containerLORA) Decode(rso *readSeekOffset) (model, error) {
var version uint32
binary.Read(rso, binary.LittleEndian, &version)

switch version {
case 1:
default:
return nil, errors.New("invalid version")
}

c.version = version

// remaining file contents aren't decoded
rso.Seek(0, io.SeekEnd)

return nil, nil
}

const (
// Magic constant for `ggml` files (unversioned).
FILE_MAGIC_GGML = 0x67676d6c
Expand Down Expand Up @@ -161,7 +135,7 @@ func DecodeGGML(r io.ReadSeeker) (*GGML, error) {
case FILE_MAGIC_GGML, FILE_MAGIC_GGMF, FILE_MAGIC_GGJT:
return nil, ErrUnsupportedFormat
case FILE_MAGIC_GGLA:
c = &containerLORA{}
c = &ContainerGGLA{}
case FILE_MAGIC_GGUF_LE:
c = &ContainerGGUF{ByteOrder: binary.LittleEndian}
case FILE_MAGIC_GGUF_BE:
Expand All @@ -171,7 +145,9 @@ func DecodeGGML(r io.ReadSeeker) (*GGML, error) {
}

model, err := c.Decode(&ro)
if err != nil {
if errors.Is(err, io.EOF) {
// noop
} else if err != nil {
return nil, err
}

Expand Down
10 changes: 7 additions & 3 deletions llm/gguf.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ type Tensor struct {
Offset uint64

// shape is the number of elements in each dimension
Shape [4]uint64
Shape []uint64

FileName string
OffsetPadding uint64
Expand Down Expand Up @@ -156,7 +156,11 @@ func (t Tensor) TypeSize() uint64 {
}

func (t Tensor) Parameters() uint64 {
return t.Shape[0] * t.Shape[1] * t.Shape[2] * t.Shape[3]
var count uint64 = 1
for _, n := range t.Shape {
count *= n
}
return count
}

func (t Tensor) Size() uint64 {
Expand Down Expand Up @@ -703,7 +707,7 @@ func (llm *GGUFModel) Decode(rso *readSeekOffset) error {
Name: name,
Kind: llm.readU32(rso),
Offset: llm.readU64(rso),
Shape: shape,
Shape: shape[:],
}

llm.Tensors = append(llm.Tensors, tensor)
Expand Down
8 changes: 7 additions & 1 deletion server/images.go
Original file line number Diff line number Diff line change
Expand Up @@ -473,7 +473,13 @@ func CreateModel(ctx context.Context, name, modelFileDir string, commands []pars
}
defer bin.Close()

layer, err := NewLayer(bin, mediatype)
ggml, err := llm.DecodeGGML(bin)
if err != nil {
return err
}

sr := io.NewSectionReader(bin, 0, ggml.Size)
layer, err := NewLayer(sr, mediatype)
if err != nil {
return err
}
Expand Down

0 comments on commit 76bdebb

Please sign in to comment.