Open ghost opened 4 years ago
Hi guys,
Hope you are all well !
I would like to read symlinked content like from the example above while cloning a repository. But, I read only the relative path. Is there a way to get the final yaml file while iterating through the worktree ?
Reference: https://github.com/onionltd/oniontree/blob/master/tagged/archive/infocon.yaml is a symlink of https://github.com/onionltd/oniontree/blob/master/unsorted/infocon.yaml
My script (to reproduce the problem):
package main import ( "bytes" "fmt" stdioutil "io/ioutil" "log" "net/http" "os" "path" "strings" "github.com/goccy/go-yaml" "github.com/gosimple/slug" "github.com/jinzhu/gorm" "github.com/k0kubun/pp" _ "github.com/mattn/go-sqlite3" "github.com/qor/admin" "gopkg.in/src-d/go-billy.v4" "gopkg.in/src-d/go-billy.v4/memfs" "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/storage/memory" "gopkg.in/src-d/go-git.v4/utils/ioutil" "github.com/onionltd/oniontree-tools/pkg/types/service" ) var ( debugMode = true debugMode2 = true isTruncate = true db *gorm.DB tables = []interface{}{ &Tag{}, &Service{}, &PublicKey{}, &URL{}, } ) // Create a GORM-backend model type Tag struct { gorm.Model Name string `gorm:"size:32;unique" json:"name" yaml:"name"` } type Service struct { gorm.Model Name string `json:"name" yaml:"name"` Slug string `json:"slug,omitempty" yaml:"slug,omitempty"` Description string `json:"description,omitempty" yaml:"description,omitempty"` URLs []*URL `json:"urls,omitempty" yaml:"urls,omitempty"` PublicKeys []*PublicKey `json:"public_keys,omitempty" yaml:"public_keys,omitempty"` Tags []*Tag `gorm:"many2many:service_tags;" json:"tags,omitempty" yaml:"tags,omitempty"` } type URL struct { gorm.Model Name string `gorm:"size:255;unique" json:"href" yaml:"href"` Healthy bool `json:"healthy" yaml:"healthy"` ServiceID uint `json:"-" yaml:"-"` } type PublicKey struct { gorm.Model UID string `gorm:"primary_key" json:"id,omitempty" yaml:"id,omitempty"` UserID string `json:"user_id,omitempty" yaml:"user_id,omitempty"` Fingerprint string `json:"fingerprint,omitempty" yaml:"fingerprint,omitempty"` Description string `json:"description,omitempty" yaml:"description,omitempty"` Value string `json:"value" yaml:"value"` ServiceID uint `json:"-" yaml:"-"` } func main() { db, _ := gorm.Open("sqlite3", "oniontree.db") if isTruncate { truncateTables(db, tables...) } db.AutoMigrate(&Tag{}, &Service{}, &URL{}, &PublicKey{}) // Initalize // ref. https://doc.getqor.com/admin/general.html Admin := admin.New(&admin.AdminConfig{ DB: db, SiteName: "OnionTreeLtd", }) // Allow to use Admin to manage Tag, PublicKey, URL, Service Admin.AddResource(&Tag{}) svc := Admin.AddResource(&Service{}) svc.Meta(&admin.Meta{ Name: "Description", Type: "rich_editor", }) pks := Admin.AddResource(&PublicKey{}) pks.Meta(&admin.Meta{ Name: "Value", Type: "text", }) Admin.AddResource(&URL{}) getWorkTree(db) // initalize an HTTP request multiplexer mux := http.NewServeMux() // Mount admin interface to mux Admin.MountTo("/admin", mux) fmt.Println("Listening on: 9000") http.ListenAndServe(":9000", mux) } // Clone a repository to memory and ... func getWorkTree(db *gorm.DB) error { fs := memfs.New() r, err := git.Clone(memory.NewStorage(), fs, &git.CloneOptions{ URL: "https://github.com/onionltd/oniontree", ReferenceName: plumbing.NewBranchReferenceName("master"), SingleBranch: true, Depth: 1, Tags: git.NoTags, }) if err != nil { return err } head, err := r.Head() if err != nil { return err } commit, err := r.CommitObject(head.Hash()) if err != nil { return err } tree, err := r.TreeObject(commit.TreeHash) if err != nil { return err } type treePath struct { *object.Tree Path string } for frontier := []treePath{{Tree: tree, Path: "/"}}; len(frontier) > 0; frontier = frontier[1:] { t := frontier[0] for _, e := range t.Entries { pp.Println("e.Name: ", e.Name, "t.Path: ", t.Path) if e.Mode != filemode.Dir { // We only care about directories. continue } if strings.HasPrefix(e.Name, ".") || strings.HasPrefix(e.Name, "_") || e.Name == "testdata" { fmt.Println("Continue because e.Name=", e.Name) continue } tree, err := r.TreeObject(e.Hash) if err != nil { fmt.Println("error with e.Hash=", e.Hash) return err } frontier = append(frontier, treePath{ Tree: tree, Path: path.Join(t.Path, e.Name), }) } i := 0 for _, e := range t.Entries { // if e.Mode != filemode.Regular && e.Mode != filemode.Executable { // fmt.Println("continue with filemode.Regular=", filemode.Regular, " or filemode.Executable", filemode.Executable) // continue // } parts := strings.Split(t.Path, "/") if len(parts) <= 2 { continue } if debugMode2 { fmt.Printf("Name:%s Path:%s Length parts:%d\n", e.Name, t.Path, len(parts)) fmt.Printf("Name:%s Path:%s Tag:%s\n", e.Name, t.Path, parts[2]) pp.Println("parts", parts) if i == 100 { os.Exit(1) } i++ } fmt.Println(path.Join(t.Path, e.Name)) blob, err := r.BlobObject(e.Hash) if err != nil { return err } r, err := blob.Reader() if err != nil { return err } svc := service.Service{} buf := new(bytes.Buffer) buf.ReadFrom(r) if e.Mode == filemode.Symlink { // pp.Println(e) oldname, err := fs.Readlink(path.Join(t.Path, e.Name)) pp.Println("oldname: ", oldname) if err != nil { return err } } // pp.Println("newStr", newStr) yaml.Unmarshal(buf.Bytes(), &svc) if debugMode { pp.Println("svc: ", svc) } if debugMode2 { pp.Println("===============================================================") pp.Println(buf.String()) pp.Println("===============================================================") } r.Close() // add service m := &Service{ Name: svc.Name, Description: svc.Description, Slug: slug.Make(svc.Name), } if err := db.Create(m).Error; err != nil { fmt.Println(err) os.Exit(1) } // add public keys for _, publicKey := range svc.PublicKeys { pubKey := &PublicKey{ UID: publicKey.ID, UserID: publicKey.UserID, Fingerprint: publicKey.Fingerprint, Description: publicKey.Description, Value: publicKey.Value, } if _, err := createOrUpdatePublicKey(db, m, pubKey); err != nil { fmt.Println(err) os.Exit(1) } } // add urls for _, url := range svc.URLs { var urlExists URL u := &URL{Name: url} if db.Where("name = ?", url).First(&urlExists).RecordNotFound() { db.Create(&u) if debugMode { pp.Println(u) } } if _, err := createOrUpdateURL(db, m, u); err != nil { fmt.Println(err) os.Exit(1) } } // add tags // check if tag already exists tag := &Tag{Name: parts[2]} var tagExists Tag if db.Where("name = ?", parts[1]).First(&tagExists).RecordNotFound() { db.Create(&tag) if debugMode { pp.Println(tag) } } if _, err := createOrUpdateTag(db, m, tag); err != nil { fmt.Println(err) os.Exit(1) } } } return nil } func checkoutFileSymlink(f *object.File, fs billy.Filesystem) (err error) { from, err := f.Reader() if err != nil { return } defer ioutil.CheckClose(from, &err) bytes, err := stdioutil.ReadAll(from) if err != nil { return } err = fs.Symlink(string(bytes), f.Name) return } func createOrUpdateTag(db *gorm.DB, svc *Service, tag *Tag) (bool, error) { var existingSvc Service if db.Where("slug = ?", svc.Slug).First(&existingSvc).RecordNotFound() { err := db.Create(svc).Error return err == nil, err } var existingTag Tag if db.Where("name = ?", tag.Name).First(&existingTag).RecordNotFound() { err := db.Create(tag).Error return err == nil, err } svc.ID = existingSvc.ID svc.CreatedAt = existingSvc.CreatedAt svc.Tags = append(svc.Tags, &existingTag) return false, db.Save(svc).Error } func findPublicKeyByUID(db *gorm.DB, uid string) *PublicKey { pubKey := &PublicKey{} if err := db.Where(&PublicKey{UID: uid}).First(pubKey).Error; err != nil { log.Fatalf("can't find public_key with uid = %q, got err %v", uid, err) } return pubKey } func createOrUpdatePublicKey(db *gorm.DB, svc *Service, pubKey *PublicKey) (bool, error) { var existingSvc Service if db.Where("slug = ?", svc.Slug).First(&existingSvc).RecordNotFound() { err := db.Create(svc).Error return err == nil, err } var existingPublicKey PublicKey if db.Where("uid = ?", pubKey.UID).First(&existingPublicKey).RecordNotFound() { err := db.Create(pubKey).Error return err == nil, err } svc.ID = existingSvc.ID svc.CreatedAt = existingSvc.CreatedAt svc.PublicKeys = append(svc.PublicKeys, &existingPublicKey) return false, db.Save(svc).Error } func createOrUpdateURL(db *gorm.DB, svc *Service, url *URL) (bool, error) { var existingSvc Service if db.Where("slug = ?", svc.Slug).First(&existingSvc).RecordNotFound() { err := db.Create(svc).Error return err == nil, err } var existingURL URL if db.Where("name = ?", url.Name).First(&existingURL).RecordNotFound() { err := db.Create(url).Error return err == nil, err } svc.ID = existingSvc.ID svc.CreatedAt = existingSvc.CreatedAt svc.URLs = append(svc.URLs, &existingURL) return false, db.Save(svc).Error } func truncateTables(db *gorm.DB, tables ...interface{}) { for _, table := range tables { if debugMode { pp.Println(table) } if err := db.DropTableIfExists(table).Error; err != nil { panic(err) } db.AutoMigrate(table) } }
Thanks in advance for your insights and inputs !
Cheers, X
Hi guys,
Hope you are all well !
I would like to read symlinked content like from the example above while cloning a repository. But, I read only the relative path. Is there a way to get the final yaml file while iterating through the worktree ?
Reference: https://github.com/onionltd/oniontree/blob/master/tagged/archive/infocon.yaml is a symlink of https://github.com/onionltd/oniontree/blob/master/unsorted/infocon.yaml
My script (to reproduce the problem):
Thanks in advance for your insights and inputs !
Cheers, X