File driver traversing subdirectories

[smferguson]

We would like to be able to traverse subdirectories when using the file driver. It seems like not doing so may have been a design decision (https://github.com/golang-migrate/migrate/tree/master/source/httpfs/testdata/sql/subdirs-are-ignored). Is there background here? Would a PR that allowed this behavior be accepted (if it were written well/had tests)?

Thanks.

[dhui]

Sub-directories are ignored for simplicity and safety. Migrations are run in order (parsed from the version number). If you were to traverse into sub-directories, the default parsing and sorting would flatten the migrations e.g. the sub-directory path is ignored

What is your desired behavior? I'm open to supporting traversing sub-directories provided it's a non-default option and the behavior is clearly documented and tested.

[smferguson]

I'd like it to continue to use the default sorting. I would just like to be able to group the migrations in nested directories by date, migrations/YYYY/MM/DD or something like that, and have the migrator pull out all the migrations. Having 1 folder with a few hundred migrations feels unwieldy (yes this is a little OCD).

[dhui]

Are you writing multiple migrations per day? If not, I don't see how hundreds of directories is different from hundreds of files.

Your usage seems very specific, so I'm hesitant to support traversing sub-directories unless there's more demand for it. e.g. other people may want the migration version parsed out differently from the path vs the base filename However, feel free to create your own source driver that does this.

[smferguson]

It wouldn't be hundreds of directories. It would be a directory per year/month (I added day above by mistake). Anyway, we'll create our own 👍

[BinaKompetisi]

@smferguson Hi, have you created your own? Can you share the code?

[smferguson]

@BinaKompetisi Hey, I never had a chance, sorry.

[BinaKompetisi]

I tried writing it myself just now. But I don't know how to integrate it to the CLI.

//go:build go1.16
// +build go1.16

package main

import (
    "errors"
    "fmt"
    "io"
    "io/fs"
    "log"
    "path"
    "slices"
    "strconv"
    "strings"

    "github.com/golang-migrate/migrate/v4/source"
)

type driver struct {
    Driver
}

func init() {
    source.Register("flatdir", &driver{})
}

// New returns a new Driver from io/fs#FS and a relative path.
func New(fsys fs.FS, path string) (source.Driver, error) {
    var i driver
    if err := i.Init(fsys, path); err != nil {
        return nil, fmt.Errorf("failed to init driver with path %s: %w", path, err)
    }
    return &i, nil
}

func (d *driver) Open(url string) (source.Driver, error) {
    return nil, errors.New("Open() cannot be called on the iofs passthrough driver")
}

type Driver struct {
    migrations *source.Migrations
    fsys       fs.FS
    path       string
}

type migrationEntry struct {
    entry fs.DirEntry
    path  string
}

// Init prepares not initialized IoFS instance to read migrations from a
// io/fs#FS instance and a relative path.
func (d *Driver) Init(fsys fs.FS, path string) error {

    var entries = getAllFiles(fsys, path)
    ms := source.NewMigrations()
    for _, e := range entries {
        if e.entry.IsDir() {
            continue
        }
        m, err := createMigration(e)
        if err != nil {
            return err
        }
        file, err := e.entry.Info()
        if err != nil {
            return err
        }
        if !ms.Append(m) {
            return source.ErrDuplicateMigration{
                Migration: *m,
                FileInfo:  file,
            }
        }
    }
    d.fsys = fsys
    d.path = path
    d.migrations = ms
    return nil
}

func getAllFiles(fsys fs.FS, path string) []migrationEntry {
    var entries []migrationEntry
    fs.WalkDir(fsys, path, func(path string, d fs.DirEntry, err error) error {
        if err != nil {
            log.Fatal(err)
        }
        if !d.IsDir() {
            entry := migrationEntry{
                entry: d,
                path:  path,
            }
            entries = append(entries, entry)
        }
        return nil
    })
    slices.SortFunc(entries, func(i, j migrationEntry) int {
        return strings.Compare(i.path, j.path)
    })
    return entries
}

func createMigration(entry migrationEntry) (*source.Migration, error) {
    m := source.Regex.FindStringSubmatch(entry.entry.Name())
    if len(m) != 5 {
        return nil, source.ErrParse

    }
    versionUint64, err := strconv.ParseUint(m[1], 10, 64)
    if err != nil {
        return nil, err
    }
    return &source.Migration{
        Version:    uint(versionUint64),
        Identifier: m[2],
        Direction:  source.Direction(m[3]),
        Raw:        entry.path,
    }, nil
}

// Close is part of source.Driver interface implementation.
// Closes the file system if possible.
func (d *Driver) Close() error {
    c, ok := d.fsys.(io.Closer)
    if !ok {
        return nil
    }
    return c.Close()
}

// First is part of source.Driver interface implementation.
func (d *Driver) First() (version uint, err error) {
    if version, ok := d.migrations.First(); ok {
        return version, nil
    }
    return 0, &fs.PathError{
        Op:   "first",
        Path: d.path,
        Err:  fs.ErrNotExist,
    }
}

// Prev is part of source.Driver interface implementation.
func (d *Driver) Prev(version uint) (prevVersion uint, err error) {
    if version, ok := d.migrations.Prev(version); ok {
        return version, nil
    }
    return 0, &fs.PathError{
        Op:   "prev for version " + strconv.FormatUint(uint64(version), 10),
        Path: d.path,
        Err:  fs.ErrNotExist,
    }
}

// Next is part of source.Driver interface implementation.
func (d *Driver) Next(version uint) (nextVersion uint, err error) {
    if version, ok := d.migrations.Next(version); ok {
        return version, nil
    }
    return 0, &fs.PathError{
        Op:   "next for version " + strconv.FormatUint(uint64(version), 10),
        Path: d.path,
        Err:  fs.ErrNotExist,
    }
}

// ReadUp is part of source.Driver interface implementation.
func (d *Driver) ReadUp(version uint) (r io.ReadCloser, identifier string, err error) {
    if m, ok := d.migrations.Up(version); ok {
        body, err := d.open(path.Join(d.path, m.Raw))
        if err != nil {
            return nil, "", err
        }
        return body, m.Identifier, nil
    }
    return nil, "", &fs.PathError{
        Op:   "read up for version " + strconv.FormatUint(uint64(version), 10),
        Path: d.path,
        Err:  fs.ErrNotExist,
    }
}

// ReadDown is part of source.Driver interface implementation.
func (d *Driver) ReadDown(version uint) (r io.ReadCloser, identifier string, err error) {
    if m, ok := d.migrations.Down(version); ok {
        body, err := d.open(path.Join(d.path, m.Raw))
        if err != nil {
            return nil, "", err
        }
        return body, m.Identifier, nil
    }
    return nil, "", &fs.PathError{
        Op:   "read down for version " + strconv.FormatUint(uint64(version), 10),
        Path: d.path,
        Err:  fs.ErrNotExist,
    }
}

func (d *Driver) open(path string) (fs.File, error) {
    f, err := d.fsys.Open(path)
    if err == nil {
        return f, nil
    }
    // Some non-standard file systems may return errors that don't include the path, that
    // makes debugging harder.
    if !errors.As(err, new(*fs.PathError)) {
        err = &fs.PathError{
            Op:   "open",
            Path: path,
            Err:  err,
        }
    }
    return nil, err
}