Skip to content

fix: avoid skipping symlinked directories during initial upload #2963

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion helper/server/tar.go
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ func tarFolder(basePath string, fileInformation *fileInformation, writtenFiles m
func tarFile(basePath string, fileInformation *fileInformation, writtenFiles map[string]bool, stat os.FileInfo, tw *tar.Writer) error {
var err error
filepath := path.Join(basePath, fileInformation.Name)
if stat.Mode()&os.ModeSymlink == os.ModeSymlink {
if fsutil.IsSymlink(stat.Mode()) {
if filepath, err = os.Readlink(filepath); err != nil {
return nil
}
Expand Down
6 changes: 4 additions & 2 deletions pkg/devspace/hook/download.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@ import (
"compress/gzip"
"context"
"fmt"
devspacecontext "github.com/loft-sh/devspace/pkg/devspace/context"
"io"
"os"
"path"
"path/filepath"
"strings"

devspacecontext "github.com/loft-sh/devspace/pkg/devspace/context"
"github.com/loft-sh/devspace/pkg/util/fsutil"

"github.com/loft-sh/devspace/pkg/devspace/config/versions/latest"
"github.com/loft-sh/devspace/pkg/devspace/kubectl"
"github.com/loft-sh/devspace/pkg/devspace/kubectl/selector"
Expand Down Expand Up @@ -146,7 +148,7 @@ func untarAll(reader io.Reader, destDir, prefix string, log logpkg.Logger) error
continue
}

if mode&os.ModeSymlink != 0 {
if fsutil.IsSymlink(mode) {
if !symlinkWarningPrinted {
symlinkWarningPrinted = true
log.Warnf("warning: skipping symlink: %q -> %q\n", destFileName, header.Linkname)
Expand Down
7 changes: 4 additions & 3 deletions pkg/devspace/hook/upload.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,14 @@ import (
"archive/tar"
"compress/gzip"
"context"
devspacecontext "github.com/loft-sh/devspace/pkg/devspace/context"
"github.com/loft-sh/devspace/pkg/util/fsutil"
"io"
"os"
"path"
"path/filepath"

devspacecontext "github.com/loft-sh/devspace/pkg/devspace/context"
"github.com/loft-sh/devspace/pkg/util/fsutil"

"github.com/loft-sh/devspace/pkg/devspace/config/versions/latest"
"github.com/loft-sh/devspace/pkg/devspace/kubectl"
"github.com/loft-sh/devspace/pkg/devspace/kubectl/selector"
Expand Down Expand Up @@ -141,7 +142,7 @@ func recursiveTar(srcBase, srcFile, destBase, destFile string, tw *tar.Writer) e
}
}
return nil
} else if stat.Mode()&os.ModeSymlink != 0 {
} else if fsutil.IsSymlink(stat.Mode()) {
//case soft link
hdr, _ := tar.FileInfoHeader(stat, fpath)
target, err := os.Readlink(fpath)
Expand Down
6 changes: 4 additions & 2 deletions pkg/devspace/sync/evaluater.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
package sync

import (
"os"

"github.com/loft-sh/devspace/helper/remote"
"github.com/loft-sh/devspace/pkg/util/fsutil"
"github.com/loft-sh/devspace/pkg/util/log"
"os"
)

// s.fileIndex needs to be locked before this function is called
Expand Down Expand Up @@ -132,7 +134,7 @@ func shouldRemoveLocal(absFilepath string, fileInformation *FileInformation, s *
}

return false
} else if stat.Mode()&os.ModeSymlink != 0 {
} else if fsutil.IsSymlink(stat.Mode()) {
return true
}

Expand Down
8 changes: 4 additions & 4 deletions pkg/devspace/sync/initial.go
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@ func (i *initialSyncer) deltaState(remoteState map[string]*FileInformation, loca
delete(remoteState, relativePath)

// should this directory be added?
if !ignore && stat.Files == 0 {
if !ignore {
i.o.FileIndex.Lock()
action := i.decide(stat, strategy)
i.o.FileIndex.Unlock()
Expand Down Expand Up @@ -291,7 +291,7 @@ func (i *initialSyncer) CalculateLocalState(absPath string, localState map[strin
if !ignore {
// Retrieve the real stat instead of the symlink one
lstat, err := os.Lstat(absPath)
if err == nil && lstat.Mode()&os.ModeSymlink != 0 {
if err == nil && fsutil.IsSymlink(lstat.Mode()) {
// Get real path
targetPath, err := filepath.EvalSymlinks(absPath)
if err != nil {
Expand Down Expand Up @@ -329,7 +329,7 @@ func (i *initialSyncer) CalculateLocalState(absPath string, localState map[strin
Size: stat.Size(),
Mode: stat.Mode(),
IsDirectory: false,
IsSymbolicLink: stat.Mode()&os.ModeSymlink != 0,
IsSymbolicLink: fsutil.IsSymlink(stat.Mode()),
ResolvedLink: isSymlink,
}
}
Expand All @@ -353,7 +353,7 @@ func (i *initialSyncer) calculateLocalDirState(absPath string, stat os.FileInfo,
Size: stat.Size(),
Mode: stat.Mode(),
IsDirectory: true,
IsSymbolicLink: stat.Mode()&os.ModeSymlink != 0,
IsSymbolicLink: fsutil.IsSymlink(stat.Mode()),
ResolvedLink: isSymlink,
Files: len(files),
}
Expand Down
28 changes: 28 additions & 0 deletions pkg/devspace/sync/sync_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -363,6 +363,7 @@ func makeBasicTestCases() (testCaseList, testCaseList) {

filesToCheck = makeSymLinkTestCases(filesToCheck)
foldersToCheck = makeSymLinkTestCases(foldersToCheck)
foldersToCheck = makeSymLinkTestCasesWithContent(foldersToCheck)

//Add Files and Folders that are inside a shared testFolder
filesToCheck = makeDeepTestCases(filesToCheck)
Expand Down Expand Up @@ -556,6 +557,33 @@ func makeSymLinkTestCases(testCases testCaseList) testCaseList {
return testCases
}

func makeSymLinkTestCasesWithContent(testCases testCaseList) testCaseList {
for _, f := range testCases {
if f.isSymLink || strings.Contains(f.path, "Remote") || f.path == "testFolder" {
continue
}

deepEquivalent := checkedFileOrFolder{
path: strings.ReplaceAll(f.path, "Folder", "SymLinkToFolderWithContents"),
shouldExistInLocal: f.shouldExistInLocal,
shouldExistInRemote: f.shouldExistInRemote,
editLocation: f.editLocation,
isSymLink: true,
}
testCases = append(testCases, deepEquivalent)

testCases = append(testCases, checkedFileOrFolder{
path: filepath.Join(deepEquivalent.path, "content"),
shouldExistInLocal: f.shouldExistInLocal,
shouldExistInRemote: f.shouldExistInRemote,
editLocation: f.editLocation,
isSymLink: false,
})
}

return testCases
}

func createTestFilesAndFolders(local string, remote string, outside string, filesToCheck testCaseList, foldersToCheck testCaseList) error {
for _, f := range foldersToCheck {
createLocation := f.editLocation
Expand Down
7 changes: 4 additions & 3 deletions pkg/devspace/sync/tar.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package sync
import (
"archive/tar"
"compress/gzip"
"github.com/loft-sh/devspace/pkg/util/fsutil"
"io"
"os"
"path"
Expand All @@ -12,6 +11,8 @@ import (
"strings"
"time"

"github.com/loft-sh/devspace/pkg/util/fsutil"

"github.com/loft-sh/devspace/helper/server/ignoreparser"

"github.com/loft-sh/devspace/pkg/util/log"
Expand Down Expand Up @@ -290,7 +291,7 @@ func (a *Archiver) tarFolder(target *FileInformation, targetStat os.FileInfo) er
func (a *Archiver) tarFile(target *FileInformation, targetStat os.FileInfo) error {
var err error
filepath := path.Join(a.basePath, target.Name)
if targetStat.Mode()&os.ModeSymlink == os.ModeSymlink {
if fsutil.IsSymlink(targetStat.Mode()) {
if filepath, err = os.Readlink(filepath); err != nil {
return nil
}
Expand Down Expand Up @@ -376,7 +377,7 @@ func fillGo18FileTypeBits(mode int64, fi os.FileInfo) int64 {
mode |= modeISREG
case fi.IsDir():
mode |= modeISDIR
case fm&os.ModeSymlink != 0:
case fsutil.IsSymlink(fm):
mode |= modeISLNK
case fm&os.ModeDevice != 0:
if fm&os.ModeCharDevice != 0 {
Expand Down
4 changes: 2 additions & 2 deletions pkg/devspace/sync/upstream.go
Original file line number Diff line number Diff line change
Expand Up @@ -505,7 +505,7 @@ func (u *upstream) evaluateChange(relativePath, fullPath string) ([]*FileInforma
// if File / Folder does not exist, we create a new remove change
// Check if symbolic link
lstat, err := os.Lstat(fullPath)
if err == nil && lstat.Mode()&os.ModeSymlink != 0 {
if err == nil && fsutil.IsSymlink(lstat.Mode()) {
_, symlinkExists := u.sync.upstream.symlinks[fullPath]

// Add symlink to map
Expand Down Expand Up @@ -539,7 +539,7 @@ func (u *upstream) evaluateChange(relativePath, fullPath string) ([]*FileInforma
Size: stat.Size(),
Mode: stat.Mode(),
IsDirectory: stat.IsDir(),
IsSymbolicLink: stat.Mode()&os.ModeSymlink != 0,
IsSymbolicLink: fsutil.IsSymlink(stat.Mode()),
}

// should we upload the file?
Expand Down
7 changes: 6 additions & 1 deletion pkg/util/fsutil/filesystem.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,16 @@ import (
recursiveCopy "github.com/otiai10/copy"
)

// IsSymlink checks if the provided file info is a symlink
func IsSymlink(mode os.FileMode) bool {
return mode&os.ModeSymlink == os.ModeSymlink
}

// IsRecursiveSymlink checks if the provided non-resolved file info
// is a recursive symlink
func IsRecursiveSymlink(f os.FileInfo, symlinkPath string) bool {
// check if recursive symlink
if f.Mode()&os.ModeSymlink == os.ModeSymlink {
if IsSymlink(f.Mode()) {
resolvedPath, err := filepath.EvalSymlinks(symlinkPath)
if err != nil || strings.HasPrefix(symlinkPath, filepath.ToSlash(resolvedPath)) {
return true
Expand Down
Loading