initial commit

cmd/dam/main.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"net/url"
+	"os"
+)
+
+var (
+	format = flag.String("format", "best", "Which quality to download")
+	debug  = flag.Bool("debug", false, "Enable debugging messages")
+)
+
+func printUsageLine() {
+	fmt.Fprintf(flag.CommandLine.Output(),
+		"Usage: %s [options] playlist-url output-file\n", flag.CommandLine.Name())
+}
+
+func main() {
+	flag.Usage = func() {
+		printUsageLine()
+		flag.PrintDefaults()
+	}
+
+	flag.Parse()
+
+	if len(flag.CommandLine.Args()) < 3 {
+		printUsageLine()
+	}
+
+	playlist := flag.CommandLine.Arg(0)
+	filename := flag.CommandLine.Arg(1)
+
+	u, err := url.Parse(playlist)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fd, err := os.Create(filename)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = downloadHLS(u, fd)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

hls.go

@@ -0,0 +1,183 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"sort"
+	"time"
+
+	"github.com/grafov/m3u8"
+)
+
+func getBestBandwidth(u *url.URL) (*m3u8.Variant, error) {
+	r, err := http.Get(u.String())
+	if err != nil {
+		return nil, err
+	}
+
+	playlist, playlistType, err := m3u8.DecodeFrom(r.Body, false)
+	if err != nil {
+		return nil, err
+	} else if playlistType != m3u8.MASTER {
+		return nil, errors.New("expected Master Playlist")
+	}
+	master := playlist.(*m3u8.MasterPlaylist)
+
+	if len(master.Variants) < 1 {
+		return nil, errors.New("no streams found")
+	} else if len(master.Variants) > 1 {
+		log.Printf("found %d variations\n", len(master.Variants))
+		sort.Slice(master.Variants, func(i, j int) bool {
+			return master.Variants[i].Bandwidth > master.Variants[j].Bandwidth
+		})
+	}
+
+	for _, v := range master.Variants {
+		if v.Iframe {
+			// not interested in I-Frame Playlists, it seems they're meant to
+			// be used to get those little thumbnails when hover over the
+			// seekbar
+			continue
+		}
+		return v, nil
+	}
+	return nil, errors.New("no streams found")
+}
+
+func downloadHLS(u *url.URL, dst io.Writer) error {
+	variant, err := getBestBandwidth(u)
+	if err != nil {
+		return err
+	}
+
+	variantURL, err := u.Parse(variant.URI)
+	if err != nil {
+		return err
+	}
+
+	fp := dst
+	// fp := ActivityWriter{dst}
+	seenMediaSequence := uint64(0)
+	byterangeOffsets := make(map[string]int64)
+
+	for {
+		log.Println("downloading playlist", variant)
+		r, err := http.Get(variantURL.String())
+		if err != nil {
+			return err
+		}
+
+		playlist, playlistType, err := DecodeFrom(r.Body, false)
+		if err != nil {
+			return err
+		} else if playlistType != m3u8.MEDIA {
+			return errors.New("expected Media Playlist")
+		}
+		media := playlist.(MediaPlaylist)
+
+		if media.TargetDuration <= 0 {
+			return errors.New("EXT-X-TARGETDURATION was not positive")
+		} else if media.TargetDuration >= 90*time.Second {
+			return errors.New("EXT-X-TARGETDURATION was too long")
+		}
+
+		if media.Iframe {
+			// as far as I can tell, "normal" Media Playlists that are not
+			// given in an EXT-X-I-FRAME-INF should not be EXT-X-I-FRAMES-ONLY
+			log.Fatal("EXT-I-FRAMES-ONLY")
+		}
+
+		mediaSequence := media.SeqNo
+		if seenMediaSequence >= mediaSequence+uint64(len(media.Segments))-1 {
+			// If the client reloads a Playlist file and finds that it has not
+			// changed, then it MUST wait for a period of one-half the target
+			// duration before retrying.
+
+			time.Sleep(media.TargetDuration / 2)
+			continue
+		}
+
+		// When a client loads a Playlist file for the first time or reloads a
+		// Playlist file and finds that it has changed since the last time it
+		// was loaded, the client MUST wait for at least the target duration
+		// before attempting to reload the Playlist file again, measured from
+		// the last time the client began loading the Playlist file.
+		waitCh := time.After(media.TargetDuration)
+
+		segCh := make(chan io.ReadCloser)
+		doneCh := make(chan struct{})
+
+		go func() {
+			for r := range segCh {
+				if _, err := io.Copy(fp, r); err != nil {
+					// return err
+					log.Fatal(err)
+				}
+				if err := r.Close(); err != nil {
+					// return err
+					log.Fatal(err)
+				}
+			}
+			close(doneCh)
+		}()
+
+		for i, seg := range media.Segments {
+			seqID := mediaSequence + uint64(i)
+			segURL, _ := variantURL.Parse(seg.URI)
+
+			if seqID <= seenMediaSequence {
+				log.Println("skipping segment", seg.URI)
+				continue
+			} else if seqID > seenMediaSequence+1 {
+				log.Printf("\033[31m%d segments expired\033[m\n", seqID-seenMediaSequence-1)
+			}
+			seenMediaSequence = seqID
+
+			if seg.Key != nil {
+				log.Fatalln("segment is encrypted")
+			}
+
+			log.Println("downloading segment", seg.URI)
+			req, err := http.NewRequest("GET", segURL.String(), nil)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			if seg.Limit < 0 {
+				return errors.New("EXT-X-BYTERANGE is negative")
+			} else if seg.Limit != 0 {
+				offset, ok := byterangeOffsets[seg.URI]
+				if seg.Offset != 0 {
+					offset = seg.Offset
+				} else if !ok {
+					return errors.New("EXT-X-BYTERANGE offset not given")
+				}
+
+				end := offset + seg.Limit - 1
+				req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, end))
+				byterangeOffsets[seg.URI] = end + 1
+			}
+
+			segData, err := http.DefaultClient.Do(req)
+			if err != nil {
+				return err
+			}
+
+			segCh <- ActivityReadCloser{segData.Body}
+		}
+
+		close(segCh)
+		<-doneCh
+
+		if media.Closed {
+			break
+		}
+		<-waitCh
+	}
+
+	return nil
+}

m3u8.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"io"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/grafov/m3u8"
+)
+
+type MediaSegment struct {
+	URI *url.URL
+
+	*m3u8.MediaSegment
+}
+
+type MediaPlaylist struct {
+	TargetDuration time.Duration
+	// Segments       []*MediaSegment
+
+	*m3u8.MediaPlaylist
+}
+
+func DecodeFrom(r io.Reader, strict bool) (playlist m3u8.Playlist, playlistType m3u8.ListType, err error) {
+	playlist, playlistType, err = m3u8.DecodeFrom(r, strict)
+
+	if playlistType == m3u8.MEDIA {
+		media := playlist.(*m3u8.MediaPlaylist)
+
+		var key *m3u8.Key
+		for i := uint(0); i < media.Count(); i++ {
+			seg := media.Segments[i]
+
+			if seg.Key != nil {
+				if strings.ToUpper(seg.Key.Method) != "NONE" {
+					key = seg.Key
+				} else {
+					key = nil
+				}
+			}
+			seg.Key = key
+		}
+		media.Segments = media.Segments[:media.Count()]
+
+		playlist = MediaPlaylist{
+			TargetDuration: time.Duration(media.TargetDuration * 1000000000),
+			MediaPlaylist:  media,
+		}
+	}
+
+	return
+}

progress.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"io"
+	"log"
+	"time"
+)
+
+var (
+	activityCh = make(chan int)
+)
+
+type ActivityReadCloser struct {
+	io.ReadCloser
+}
+
+func (p ActivityReadCloser) Read(buf []byte) (int, error) {
+	read, err := p.ReadCloser.Read(buf)
+	activityCh <- read
+	return read, err
+}
+
+type ActivityWriter struct {
+	io.Writer
+}
+
+func (p ActivityWriter) Write(buf []byte) (int, error) {
+	written, err := p.Writer.Write(buf)
+	activityCh <- written
+	return written, err
+}
+
+func init() {
+	go func() {
+		second := time.Tick(time.Second)
+		activity := 0
+
+		for {
+			select {
+			case <-second:
+				log.Printf("DL @ %d kB/s", activity/1024)
+				activity = 0
+
+			case x := <-activityCh:
+				activity += x
+			}
+		}
+	}()
+}