feat: enhance archiving and extraction capabilities with additional compression formats and ownership preservation (#13458)

* feat: enhance archiving and extraction capabilities with additional compression formats and ownership preservation

* feat: enhance archiving and extraction capabilities with additional compression formats and ownership preservation

* feat: enhance archiving and extraction capabilities with additional compression formats and ownership preservation
This commit is contained in:
蘭 authored and GitHub committed 2026-08-04 17:34:50 +08:00
1 parent 1eb429631d
commit 91ae846418
10 files changed
+655 -114

No files matched your search

+51 -6
View File
@@ -16,6 +16,7 @@ import (
"sort"
"strconv"
"strings"
"syscall"
"time"
"unicode/utf8"
@@ -466,7 +467,7 @@ func (f *FileService) Compress(c request.FileCompress) error {
func preflightCompressTool(compressType files.CompressType) error {
switch compressType {
case files.TarGz, files.Rar, files.X7z:
case files.Tar, files.Gz, files.Bz2, files.TarBz2, files.Tgz, files.TarGz, files.Xz, files.TarXz, files.Rar, files.X7z:
_, err := files.NewShellArchiver(compressType)
return err
default:
@@ -476,7 +477,7 @@ func preflightCompressTool(compressType files.CompressType) error {
func preflightDecompressTool(decompressType files.CompressType) error {
switch decompressType {
case files.Rar, files.X7z:
case files.Rar:
_, err := files.NewExtractShellArchiver(decompressType)
return err
default:
@@ -533,7 +534,10 @@ func (f *FileService) DeCompress(c request.FileDeCompress) error {
_ = os.RemoveAll(c.Dst)
}
}()
if err := fo.Decompress(t.TaskCtx, c.Path, tempDst, files.CompressType(c.Type), c.Secret); err != nil {
if err := fo.DecompressWithOptions(t.TaskCtx, c.Path, tempDst, files.CompressType(c.Type), c.Secret, files.DecompressOptions{
PreserveOwner: true,
AllowCLIReextract: true,
}); err != nil {
return err
}
if err := fo.CreateDir(c.Dst, constant.DirPerm); err != nil {
@@ -551,19 +555,42 @@ func (f *FileService) DeCompress(c request.FileDeCompress) error {
}
func copyDecompressTree(ctx context.Context, srcDir, dstDir string) error {
state := decompressCopyState{hardlinks: make(map[decompressFileIdentity]string)}
entries, err := os.ReadDir(srcDir)
if err != nil {
return err
}
for _, entry := range entries {
if err := copyDecompressEntry(ctx, filepath.Join(srcDir, entry.Name()), filepath.Join(dstDir, entry.Name())); err != nil {
if err := copyDecompressEntryWithState(ctx, filepath.Join(srcDir, entry.Name()), filepath.Join(dstDir, entry.Name()), &state); err != nil {
return err
}
}
return nil
}
type decompressFileIdentity struct {
device uint64
inode uint64
}
type decompressCopyState struct {
hardlinks map[decompressFileIdentity]string
}
func decompressHardlinkIdentity(info os.FileInfo) (decompressFileIdentity, bool) {
stat, ok := info.Sys().(*syscall.Stat_t)
if !ok || stat.Nlink < 2 {
return decompressFileIdentity{}, false
}
return decompressFileIdentity{device: uint64(stat.Dev), inode: uint64(stat.Ino)}, true
}
func copyDecompressEntry(ctx context.Context, srcPath, dstPath string) (retErr error) {
state := decompressCopyState{hardlinks: make(map[decompressFileIdentity]string)}
return copyDecompressEntryWithState(ctx, srcPath, dstPath, &state)
}
func copyDecompressEntryWithState(ctx context.Context, srcPath, dstPath string, state *decompressCopyState) (retErr error) {
if err := ctx.Err(); err != nil {
return err
}
@@ -605,13 +632,16 @@ func copyDecompressEntry(ctx context.Context, srcPath, dstPath string) (retErr e
if err := applyDecompressOwnership(srcPath, dstPath); err != nil {
return err
}
if err := os.Chmod(dstPath, info.Mode().Perm()); err != nil {
return err
}
}
entries, err := os.ReadDir(srcPath)
if err != nil {
return err
}
for _, entry := range entries {
if err := copyDecompressEntry(ctx, filepath.Join(srcPath, entry.Name()), filepath.Join(dstPath, entry.Name())); err != nil {
if err := copyDecompressEntryWithState(ctx, filepath.Join(srcPath, entry.Name()), filepath.Join(dstPath, entry.Name()), state); err != nil {
return err
}
}
@@ -634,6 +664,15 @@ func copyDecompressEntry(ctx context.Context, srcPath, dstPath string) (retErr e
if err := os.MkdirAll(filepath.Dir(dstPath), constant.DirPerm); err != nil {
return err
}
identity, isHardlink := decompressHardlinkIdentity(info)
if !keepExistingFile && isHardlink {
if existingPath, ok := state.hardlinks[identity]; ok {
if err := os.Link(existingPath, dstPath); err != nil {
return err
}
return os.Chtimes(dstPath, info.ModTime(), info.ModTime())
}
}
srcFile, err := os.Open(srcPath)
if err != nil {
@@ -658,6 +697,12 @@ func copyDecompressEntry(ctx context.Context, srcPath, dstPath string) (retErr e
if err := applyDecompressOwnership(srcPath, dstPath); err != nil {
return err
}
if err := os.Chmod(dstPath, info.Mode().Perm()); err != nil {
return err
}
if isHardlink {
state.hardlinks[identity] = dstPath
}
}
return os.Chtimes(dstPath, info.ModTime(), info.ModTime())
}
@@ -667,7 +712,7 @@ func applyDecompressOwnership(srcPath, dstPath string) error {
if err != nil {
return err
}
stat, ok := info.Sys().(*unix.Stat_t)
stat, ok := info.Sys().(*syscall.Stat_t)
if !ok {
return nil
}
+2 -2
View File
@@ -14,7 +14,7 @@ type ShellArchiver interface {
func NewShellArchiver(compressType CompressType) (ShellArchiver, error) {
switch compressType {
case Tar:
case Tar, Gz, Bz2, TarBz2, Tgz, Xz, TarXz:
if err := checkCmdAvailability("tar"); err != nil {
return nil, err
}
@@ -46,7 +46,7 @@ func NewShellArchiver(compressType CompressType) (ShellArchiver, error) {
func NewExtractShellArchiver(compressType CompressType) (ShellArchiver, error) {
switch compressType {
case Tar:
case Tar, Gz, Bz2, TarBz2, Tgz, Xz, TarXz:
if err := checkCmdAvailability("tar"); err != nil {
return nil, err
}
+495 -87
View File
@@ -1,11 +1,13 @@
package files
import (
"archive/tar"
"archive/zip"
"bufio"
"compress/gzip"
"context"
"crypto/tls"
"encoding/binary"
"encoding/json"
"fmt"
"io"
@@ -20,6 +22,7 @@ import (
"strconv"
"strings"
"sync"
"syscall"
"time"
"github.com/1Panel-dev/1Panel/agent/buserr"
@@ -40,6 +43,7 @@ import (
const (
cmdDefaultTimeout = 10 * time.Second
cmdRecursiveTimeout = 5 * time.Minute
maxArchiveSymlinkTargetSize = 4 * 1024
)
var protectedPaths = []string{
@@ -953,6 +957,12 @@ func (f FileOp) Compress(ctx context.Context, srcRiles []string, dst string, nam
}
_ = f.DeleteFile(dstFile)
return NewZipArchiver().Compress(ctx, srcRiles, dstFile, "")
case Tar, Gz, Bz2, TarBz2, Tgz, Xz, TarXz:
err = NewTarArchiver(cType).Compress(ctx, srcRiles, dstFile, secret)
if err != nil {
_ = f.DeleteFile(dstFile)
return err
}
case TarGz:
err = NewTarGzArchiver().Compress(ctx, srcRiles, dstFile, secret)
if err != nil {
@@ -1031,20 +1041,217 @@ func decodeGBK(input string) (string, error) {
return decoded, nil
}
func (f FileOp) decompressWithSDK(ctx context.Context, srcFile string, dst string, cType CompressType) error {
format := getFormat(cType)
if cType == Gz {
if err := f.tryDecompressTarGz(ctx, srcFile, dst, format); err == nil {
type DecompressOptions struct {
PreserveOwner bool
AllowCLIReextract bool
}
type archiveOwnership struct {
uid int
gid int
}
func getArchiveOwnership(file archiver.File) (archiveOwnership, bool) {
header, ok := getArchiveTarHeader(file)
if ok && header.Uid >= 0 && header.Gid >= 0 {
return archiveOwnership{uid: header.Uid, gid: header.Gid}, true
}
return getArchiveZipOwnership(file)
}
const zipUnixOwnershipExtraID = 0x7875
func getArchiveZipOwnership(file archiver.File) (archiveOwnership, bool) {
var extra []byte
switch header := file.Header.(type) {
case *zip.FileHeader:
if header != nil {
extra = header.Extra
}
case zip.FileHeader:
extra = header.Extra
case *cZip.FileHeader:
if header != nil {
extra = header.Extra
}
case cZip.FileHeader:
extra = header.Extra
default:
return archiveOwnership{}, false
}
for len(extra) >= 4 {
fieldID := binary.LittleEndian.Uint16(extra[:2])
fieldSize := int(binary.LittleEndian.Uint16(extra[2:4]))
extra = extra[4:]
if fieldSize > len(extra) {
return archiveOwnership{}, false
}
field := extra[:fieldSize]
extra = extra[fieldSize:]
if fieldID != zipUnixOwnershipExtraID || len(field) < 4 || field[0] != 1 {
continue
}
uidSize := int(field[1])
if uidSize == 0 || uidSize > 4 || len(field) < 2+uidSize+1 {
return archiveOwnership{}, false
}
uid := decodeZipOwnershipID(field[2 : 2+uidSize])
gidSizeOffset := 2 + uidSize
gidSize := int(field[gidSizeOffset])
if gidSize == 0 || gidSize > 4 || len(field) < gidSizeOffset+1+gidSize {
return archiveOwnership{}, false
}
gid := decodeZipOwnershipID(field[gidSizeOffset+1 : gidSizeOffset+1+gidSize])
return archiveOwnership{uid: int(uid), gid: int(gid)}, true
}
return archiveOwnership{}, false
}
func decodeZipOwnershipID(value []byte) uint32 {
var result uint32
for i := len(value) - 1; i >= 0; i-- {
result = result<<8 | uint32(value[i])
}
return result
}
func appendZipOwnershipExtra(extra []byte, ownership archiveOwnership) []byte {
const fieldSize = 11
field := make([]byte, 4+fieldSize)
binary.LittleEndian.PutUint16(field[:2], zipUnixOwnershipExtraID)
binary.LittleEndian.PutUint16(field[2:4], fieldSize)
field[4] = 1
field[5] = 4
binary.LittleEndian.PutUint32(field[6:10], uint32(ownership.uid))
field[10] = 4
binary.LittleEndian.PutUint32(field[11:15], uint32(ownership.gid))
return append(extra, field...)
}
func getFileOwnership(info fs.FileInfo) (archiveOwnership, bool) {
stat, ok := info.Sys().(*syscall.Stat_t)
if !ok {
return archiveOwnership{}, false
}
return archiveOwnership{uid: int(stat.Uid), gid: int(stat.Gid)}, true
}
func getArchiveTarHeader(file archiver.File) (tar.Header, bool) {
switch header := file.Header.(type) {
case *tar.Header:
if header == nil {
return tar.Header{}, false
}
return *header, true
case tar.Header:
return header, true
default:
return tar.Header{}, false
}
}
func applyArchiveOwnership(filePath string, mode fs.FileMode, ownership archiveOwnership) error {
if mode&fs.ModeSymlink != 0 {
return os.Lchown(filePath, ownership.uid, ownership.gid)
}
return os.Chown(filePath, ownership.uid, ownership.gid)
}
func archiveDestinationPath(dst, name string) (string, error) {
cleaned := filepath.Clean(filepath.FromSlash(name))
if cleaned == "." {
return dst, nil
}
if filepath.IsAbs(cleaned) || cleaned == ".." || strings.HasPrefix(cleaned, ".."+string(os.PathSeparator)) {
return "", fmt.Errorf("invalid archive path: %s", name)
}
return filepath.Join(dst, cleaned), nil
}
func ensureArchiveDirectory(dirPath string, mode fs.FileMode) error {
info, err := os.Lstat(dirPath)
if err == nil {
if info.Mode()&fs.ModeSymlink != 0 || !info.IsDir() {
return fmt.Errorf("archive directory path is not a directory: %s", dirPath)
}
return nil
}
return f.DecompressGzFile(ctx, srcFile, dst)
if !os.IsNotExist(err) {
return err
}
return os.Mkdir(dirPath, mode.Perm())
}
func ensureArchiveParent(dst, filePath string) error {
root := filepath.Clean(dst)
parent := filepath.Dir(filePath)
rel, err := filepath.Rel(root, parent)
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
return fmt.Errorf("archive parent escapes destination: %s", parent)
}
if err := ensureArchiveDirectory(root, constant.DirPerm); err != nil {
return err
}
if rel == "." {
return nil
}
current := root
for _, part := range strings.Split(rel, string(os.PathSeparator)) {
current = filepath.Join(current, part)
if err := ensureArchiveDirectory(current, constant.DirPerm); err != nil {
return err
}
}
return nil
}
func validateArchiveHardlinkTarget(dst, targetPath string) error {
root := filepath.Clean(dst)
rel, err := filepath.Rel(root, targetPath)
if err != nil || rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
return fmt.Errorf("archive hardlink target escapes destination: %s", targetPath)
}
current := root
parts := strings.Split(rel, string(os.PathSeparator))
for i, part := range parts {
current = filepath.Join(current, part)
info, err := os.Lstat(current)
if err != nil {
return fmt.Errorf("archive hardlink target is unavailable: %s: %w", targetPath, err)
}
if info.Mode()&fs.ModeSymlink != 0 {
return fmt.Errorf("archive hardlink target contains a symlink: %s", current)
}
if i < len(parts)-1 && !info.IsDir() {
return fmt.Errorf("archive hardlink target parent is not a directory: %s", current)
}
if i == len(parts)-1 && !info.Mode().IsRegular() {
return fmt.Errorf("archive hardlink target is not a regular file: %s", current)
}
}
return nil
}
func (f FileOp) extractArchiveWithSDK(ctx context.Context, input io.Reader, dst string, extractor archiver.Extractor, options DecompressOptions) (bool, error) {
type dirEntry struct {
path string
mode fs.FileMode
modTime time.Time
ownership archiveOwnership
hasOwnership bool
}
type hardlinkEntry struct {
path string
target string
mode fs.FileMode
modTime time.Time
ownership archiveOwnership
hasOwnership bool
}
var dirs []dirEntry
var hardlinks []hardlinkEntry
extractionStarted := false
root := filepath.Clean(dst)
handler := func(ctx context.Context, archFile archiver.File) error {
info := archFile.FileInfo
@@ -1061,67 +1268,320 @@ func (f FileOp) decompressWithSDK(ctx context.Context, srcFile string, dst strin
}
}
}
filePath := filepath.Join(dst, fileName)
header, hasTarHeader := getArchiveTarHeader(archFile)
ownership, hasOwnership := getArchiveOwnership(archFile)
filePath, err := archiveDestinationPath(dst, fileName)
if err != nil {
return err
}
extractionStarted = true
if archFile.FileInfo.IsDir() {
if err := f.Fs.MkdirAll(filePath, info.Mode()); err != nil {
if filePath == root {
return ensureArchiveDirectory(root, constant.DirPerm)
}
if err := ensureArchiveParent(dst, filePath); err != nil {
return err
}
dirs = append(dirs, dirEntry{path: filePath, modTime: info.ModTime()})
if err := ensureArchiveDirectory(filePath, info.Mode()); err != nil {
return err
}
dirs = append(dirs, dirEntry{
path: filePath,
mode: info.Mode(),
modTime: info.ModTime(),
ownership: ownership,
hasOwnership: hasOwnership,
})
return nil
} else {
parentDir := path.Dir(filePath)
if !f.Stat(parentDir) {
if err := f.Fs.MkdirAll(parentDir, constant.DirPerm); err != nil {
}
if hasTarHeader && header.Typeflag == tar.TypeLink {
target, err := archiveDestinationPath(dst, header.Linkname)
if err != nil {
return err
}
hardlinks = append(hardlinks, hardlinkEntry{
path: filePath,
target: target,
mode: info.Mode(),
modTime: info.ModTime(),
ownership: ownership,
hasOwnership: hasOwnership,
})
return nil
}
if err := ensureArchiveParent(dst, filePath); err != nil {
return err
}
if info.Mode()&fs.ModeSymlink != 0 || hasTarHeader && header.Typeflag == tar.TypeSymlink {
target := archFile.LinkTarget
if target == "" && hasTarHeader {
target = header.Linkname
}
if target == "" && archFile.Open != nil {
fr, err := archFile.Open()
if err != nil {
return err
}
defer fr.Close()
fw, err := f.Fs.OpenFile(filePath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, info.Mode())
data, readErr := io.ReadAll(io.LimitReader(fr, maxArchiveSymlinkTargetSize+1))
closeErr := fr.Close()
if readErr != nil {
return readErr
}
if closeErr != nil {
return closeErr
}
target = string(data)
}
if len(target) > maxArchiveSymlinkTargetSize {
return fmt.Errorf("archive symlink target for %s exceeds %d bytes", fileName, maxArchiveSymlinkTargetSize)
}
if target == "" {
return fmt.Errorf("archive symlink %s has no target", fileName)
}
if err := os.RemoveAll(filePath); err != nil {
return err
}
if err := os.Symlink(target, filePath); err != nil {
return err
}
if options.PreserveOwner && hasOwnership {
if err := applyArchiveOwnership(filePath, info.Mode(), ownership); err != nil {
return fmt.Errorf("restore archive ownership for %s: %w", fileName, err)
}
}
return nil
}
if existing, err := os.Lstat(filePath); err == nil {
if existing.IsDir() {
return fmt.Errorf("archive file path is a directory: %s", fileName)
}
if err := os.Remove(filePath); err != nil {
return err
}
} else if !os.IsNotExist(err) {
return err
}
fr, err := archFile.Open()
if err != nil {
return err
}
defer fw.Close()
if _, err := io.Copy(fw, fr); err != nil {
fw, err := f.Fs.OpenFile(filePath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, info.Mode().Perm())
if err != nil {
_ = fr.Close()
return err
}
_, copyErr := io.Copy(fw, fr)
closeReadErr := fr.Close()
closeWriteErr := fw.Close()
if copyErr != nil {
return copyErr
}
if closeReadErr != nil {
return closeReadErr
}
if closeWriteErr != nil {
return closeWriteErr
}
if options.PreserveOwner && hasOwnership {
if err := applyArchiveOwnership(filePath, info.Mode(), ownership); err != nil {
return fmt.Errorf("restore archive ownership for %s: %w", fileName, err)
}
}
if err := f.Fs.Chmod(filePath, info.Mode().Perm()); err != nil {
return fmt.Errorf("restore archive mode for %s: %w", fileName, err)
}
_ = os.Chtimes(filePath, info.ModTime(), info.ModTime())
return nil
}
if err := extractor.Extract(ctx, input, nil, handler); err != nil {
return extractionStarted, err
}
for _, link := range hardlinks {
if err := ensureArchiveParent(dst, link.path); err != nil {
return extractionStarted, err
}
if err := validateArchiveHardlinkTarget(dst, link.target); err != nil {
return extractionStarted, err
}
if err := os.RemoveAll(link.path); err != nil {
return extractionStarted, err
}
if err := os.Link(link.target, link.path); err != nil {
return extractionStarted, fmt.Errorf("restore archive hardlink %s: %w", link.path, err)
}
if options.PreserveOwner && link.hasOwnership {
if err := applyArchiveOwnership(link.path, link.mode, link.ownership); err != nil {
return extractionStarted, fmt.Errorf("restore archive ownership for %s: %w", link.path, err)
}
}
if err := f.Fs.Chmod(link.path, link.mode.Perm()); err != nil {
return extractionStarted, fmt.Errorf("restore archive mode for %s: %w", link.path, err)
}
_ = os.Chtimes(link.path, link.modTime, link.modTime)
}
for i := len(dirs) - 1; i >= 0; i-- {
if options.PreserveOwner && dirs[i].hasOwnership {
if err := applyArchiveOwnership(dirs[i].path, dirs[i].mode, dirs[i].ownership); err != nil {
return extractionStarted, fmt.Errorf("restore archive ownership for %s: %w", dirs[i].path, err)
}
}
if err := f.Fs.Chmod(dirs[i].path, dirs[i].mode.Perm()); err != nil {
return extractionStarted, fmt.Errorf("restore archive mode for %s: %w", dirs[i].path, err)
}
_ = os.Chtimes(dirs[i].path, dirs[i].modTime, dirs[i].modTime)
}
return extractionStarted, nil
}
func (f FileOp) decompressWithSDKState(ctx context.Context, srcFile string, dst string, cType CompressType, secret string, options DecompressOptions) (bool, error) {
input, err := f.Fs.Open(srcFile)
if err != nil {
return false, err
}
var extractor archiver.Extractor = getFormat(cType)
if cType == X7z {
extractor = archiver.SevenZip{Password: secret}
}
extractionStarted, extractErr := f.extractArchiveWithSDK(ctx, input, dst, extractor, options)
closeErr := input.Close()
if cType == Gz {
if extractErr == nil && extractionStarted {
return true, closeErr
}
if extractionStarted {
return true, extractErr
}
return false, f.DecompressGzFile(ctx, srcFile, dst)
}
if extractErr != nil {
return extractionStarted, extractErr
}
if closeErr != nil {
return extractionStarted, closeErr
}
return extractionStarted, nil
}
func (f FileOp) decompressWithSDK(ctx context.Context, srcFile string, dst string, cType CompressType, secret string, options DecompressOptions) error {
_, err := f.decompressWithSDKState(ctx, srcFile, dst, cType, secret, options)
return err
}
func resetArchiveFallbackDestination(dst string) error {
info, err := os.Lstat(dst)
if os.IsNotExist(err) {
return os.MkdirAll(dst, constant.DirPerm)
}
if err != nil {
return err
}
defer input.Close()
if err := format.Extract(ctx, input, nil, handler); err != nil {
if info.Mode()&fs.ModeSymlink != 0 || !info.IsDir() {
return fmt.Errorf("archive fallback destination is not a directory: %s", dst)
}
mode := info.Mode().Perm()
ownership, hasOwnership := getFileOwnership(info)
if err := os.RemoveAll(dst); err != nil {
return err
}
for i := len(dirs) - 1; i >= 0; i-- {
_ = os.Chtimes(dirs[i].path, dirs[i].modTime, dirs[i].modTime)
if err := os.MkdirAll(dst, mode); err != nil {
return err
}
if hasOwnership {
if err := os.Chown(dst, ownership.uid, ownership.gid); err != nil {
return err
}
}
return os.Chmod(dst, mode)
}
func (f FileOp) decompressSevenZipWithFallback(ctx context.Context, srcFile, dst, secret string, options DecompressOptions) error {
return f.decompressSevenZipWithFallbackUsing(ctx, srcFile, dst, secret, options, f.decompressWithSDKState)
}
func (f FileOp) decompressSevenZipWithFallbackUsing(
ctx context.Context,
srcFile, dst, secret string,
options DecompressOptions,
sdkExtract func(context.Context, string, string, CompressType, string, DecompressOptions) (bool, error),
) error {
extractionStarted, sdkErr := sdkExtract(ctx, srcFile, dst, X7z, secret, options)
if sdkErr == nil {
return nil
}
if secret != "" {
return sdkErr
}
if extractionStarted && !options.AllowCLIReextract {
return sdkErr
}
if extractionStarted {
if err := resetArchiveFallbackDestination(dst); err != nil {
return fmt.Errorf("reset 7z CLI fallback destination: %w", err)
}
}
shellArchiver, err := NewExtractShellArchiver(X7z)
if err != nil {
return err
}
if global.LOG != nil {
global.LOG.Warnf("7z SDK decompression failed, falling back to CLI: %v", sdkErr)
}
if err := shellArchiver.Extract(ctx, srcFile, dst, secret); err != nil {
return fmt.Errorf("7z SDK decompression failed: %v; CLI fallback failed: %w", sdkErr, err)
}
return nil
}
type ownershipPreservingShellExtractor interface {
ExtractWithOptions(ctx context.Context, filePath, dstDir, secret string, preserveOwner bool) error
}
func extractWithShellOptions(ctx context.Context, shellArchiver ShellArchiver, srcFile, dst, secret string, options DecompressOptions) error {
if options.PreserveOwner {
if extractor, ok := shellArchiver.(ownershipPreservingShellExtractor); ok {
return extractor.ExtractWithOptions(ctx, srcFile, dst, secret, true)
}
}
return shellArchiver.Extract(ctx, srcFile, dst, secret)
}
func (f FileOp) Decompress(ctx context.Context, srcFile string, dst string, cType CompressType, secret string) error {
if cType == Tar || cType == Zip || cType == TarGz || cType == Rar || cType == X7z {
return f.DecompressWithOptions(ctx, srcFile, dst, cType, secret, DecompressOptions{})
}
func (f FileOp) DecompressWithOptions(ctx context.Context, srcFile string, dst string, cType CompressType, secret string, options DecompressOptions) error {
if cType == X7z && options.PreserveOwner {
return f.decompressSevenZipWithFallback(ctx, srcFile, dst, secret, options)
}
var shellErr error
useShell := cType == Rar || cType == Zip || cType == Tar || cType == TarGz ||
!options.PreserveOwner && cType == X7z
if useShell {
shellArchiver, err := NewExtractShellArchiver(cType)
if !f.Stat(dst) {
_ = f.CreateDir(dst, 0755)
}
if err == nil {
if err = shellArchiver.Extract(ctx, srcFile, dst, secret); err == nil {
if err = extractWithShellOptions(ctx, shellArchiver, srcFile, dst, secret, options); err == nil {
return nil
}
shellErr = err
if cType == TarGz {
if strings.Contains(err.Error(), "bad decrypt") {
return buserr.New("ErrBadDecrypt")
}
if err := shellArchiver.Extract(ctx, srcFile, dst, "-"); strings.Contains(err.Error(), "bad decrypt") {
if retryErr := extractWithShellOptions(ctx, shellArchiver, srcFile, dst, "-", options); retryErr == nil {
return nil
} else if strings.Contains(retryErr.Error(), "bad decrypt") {
return buserr.New("ErrBadDecrypt")
} else {
shellErr = retryErr
}
}
} else {
@@ -1130,7 +1590,15 @@ func (f FileOp) Decompress(ctx context.Context, srcFile string, dst string, cTyp
}
}
}
return f.decompressWithSDK(ctx, srcFile, dst, cType)
if shellErr != nil && global.LOG != nil {
global.LOG.Warnf("shell decompression for %s failed, falling back to SDK: %v", cType, shellErr)
}
if shellErr != nil && options.AllowCLIReextract {
if err := resetArchiveFallbackDestination(dst); err != nil {
return fmt.Errorf("reset %s SDK fallback destination: %w", cType, err)
}
}
return f.decompressWithSDK(ctx, srcFile, dst, cType, secret, options)
}
func ZipFile(ctx context.Context, files []archiver.File, dst afero.File, progress func(current, total int, message string)) error {
@@ -1152,6 +1620,9 @@ func ZipFile(ctx context.Context, files []archiver.File, dst afero.File, progres
}
hdr.Method = zip.Deflate
hdr.Name = file.NameInArchive
if ownership, ok := getFileOwnership(file.FileInfo); ok {
hdr.Extra = appendZipOwnershipExtra(hdr.Extra, ownership)
}
if file.IsDir() {
if !strings.HasSuffix(hdr.Name, "/") {
hdr.Name += "/"
@@ -1209,69 +1680,6 @@ func (r *contextReader) Read(p []byte) (int, error) {
}
}
func (f FileOp) tryDecompressTarGz(ctx context.Context, srcFile string, dst string, format archiver.CompressedArchive) error {
input, err := f.Fs.Open(srcFile)
if err != nil {
return err
}
defer input.Close()
type dirEntry struct {
path string
modTime time.Time
}
var dirs []dirEntry
extracted := false
handler := func(ctx context.Context, archFile archiver.File) error {
info := archFile.FileInfo
if isIgnoreFile(archFile.Name()) {
return nil
}
filePath := filepath.Join(dst, archFile.NameInArchive)
if info.IsDir() {
if err := f.Fs.MkdirAll(filePath, info.Mode()); err != nil {
return err
}
dirs = append(dirs, dirEntry{path: filePath, modTime: info.ModTime()})
} else {
parentDir := filepath.Dir(filePath)
if !f.Stat(parentDir) {
if err := f.Fs.MkdirAll(parentDir, constant.DirPerm); err != nil {
return err
}
}
fr, err := archFile.Open()
if err != nil {
return err
}
defer fr.Close()
fw, err := f.Fs.OpenFile(filePath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, info.Mode())
if err != nil {
return err
}
defer fw.Close()
if _, err := io.Copy(fw, fr); err != nil {
return err
}
_ = os.Chtimes(filePath, info.ModTime(), info.ModTime())
}
extracted = true
return nil
}
if err := format.Extract(ctx, input, nil, handler); err != nil {
return err
}
if !extracted {
return fmt.Errorf("no files extracted as tar.gz")
}
for i := len(dirs) - 1; i >= 0; i-- {
_ = os.Chtimes(dirs[i].path, dirs[i].modTime, dirs[i].modTime)
}
return nil
}
func (f FileOp) DecompressGzFile(ctx context.Context, srcFile, dst string) error {
var archiveModTime time.Time
if st, err := f.Fs.Stat(srcFile); err == nil {
+11 -2
View File
@@ -20,10 +20,19 @@ func NewRarArchiver() ShellArchiver {
}
func (z RarArchiver) Extract(ctx context.Context, filePath, dstDir string, _ string) error {
return z.ExtractWithOptions(ctx, filePath, dstDir, "", false)
}
func (z RarArchiver) ExtractWithOptions(ctx context.Context, filePath, dstDir, _ string, preserveOwner bool) error {
if err := checkCmdAvailability("unrar"); err != nil {
return err
}
return cmd.NewCommandMgr(cmd.WithContext(ctx)).Run("unrar", "x", "-y", "-o+", filePath, dstDir)
args := []string{"x", "-y", "-o+"}
if preserveOwner {
args = append(args, "-oh", "-ol", "-ow")
}
args = append(args, filePath, dstDir)
return cmd.NewCommandMgr(cmd.WithContext(ctx)).Run("unrar", args...)
}
func (z RarArchiver) Compress(ctx context.Context, sourcePaths []string, dstFile string, _ string) (err error) {
@@ -45,7 +54,7 @@ func (z RarArchiver) Compress(ctx context.Context, sourcePaths []string, dstFile
relativePaths[i] = path.Base(sp)
}
cmdArgs := append([]string{"a", "-r", tmpFile}, relativePaths...)
cmdArgs := append([]string{"a", "-r", "-oh", "-ol", "-ow", tmpFile}, relativePaths...)
cmdMgr := cmd.NewCommandMgr(cmd.WithWorkDir(baseDir), cmd.WithContext(ctx))
if err = cmdMgr.Run("rar", cmdArgs...); err != nil {
return err
+46 -8
View File
@@ -2,6 +2,8 @@ package files
import (
"context"
"fmt"
"path/filepath"
"github.com/1Panel-dev/1Panel/agent/utils/cmd"
)
@@ -19,21 +21,57 @@ func NewTarArchiver(compressType CompressType) ShellArchiver {
}
func (t TarArchiver) Extract(ctx context.Context, FilePath string, dstDir string, secret string) error {
return cmd.NewCommandMgr(cmd.WithContext(ctx)).Run(t.Cmd, t.getOptionStr("extract"), FilePath, "-C", dstDir)
return t.ExtractWithOptions(ctx, FilePath, dstDir, secret, false)
}
func (t TarArchiver) Compress(ctx context.Context, sourcePaths []string, dstFile string, secret string) error {
return nil
func (t TarArchiver) ExtractWithOptions(ctx context.Context, filePath, dstDir, _ string, preserveOwner bool) error {
if err := checkCmdAvailability(t.Cmd); err != nil {
return err
}
args := []string{t.getOptionStr("extract"), filePath, "-C", dstDir}
if preserveOwner {
args = append([]string{"--same-owner", "--same-permissions"}, args...)
}
return cmd.NewCommandMgr(cmd.WithContext(ctx)).Run(t.Cmd, args...)
}
func (t TarArchiver) getOptionStr(Option string) string {
func (t TarArchiver) Compress(ctx context.Context, sourcePaths []string, dstFile string, _ string) error {
if len(sourcePaths) == 0 {
return fmt.Errorf("source paths cannot be empty")
}
if err := checkCmdAvailability(t.Cmd); err != nil {
return err
}
baseDir := filepath.Dir(sourcePaths[0])
args := []string{t.getOptionStr("compress"), dstFile, "-C", baseDir}
for _, sourcePath := range sourcePaths {
args = append(args, filepath.Base(sourcePath))
}
return cmd.NewCommandMgr(cmd.WithContext(ctx)).Run(t.Cmd, args...)
}
func (t TarArchiver) getOptionStr(option string) string {
switch t.CompressType {
case Tar:
if Option == "compress" {
return "cvf"
} else {
return "xf"
if option == "compress" {
return "-cf"
}
return "-xf"
case Gz, Tgz, TarGz:
if option == "compress" {
return "-zcf"
}
return "-zxf"
case Bz2, TarBz2:
if option == "compress" {
return "-jcf"
}
return "-jxf"
case Xz, TarXz:
if option == "compress" {
return "-Jcf"
}
return "-Jxf"
}
return ""
}
+25 -5
View File
@@ -21,14 +21,18 @@ func NewTarGzArchiver() ShellArchiver {
}
func (t TarGzArchiver) Extract(ctx context.Context, filePath, dstDir string, secret string) error {
return t.ExtractWithOptions(ctx, filePath, dstDir, secret, false)
}
func (t TarGzArchiver) ExtractWithOptions(ctx context.Context, filePath, dstDir, secret string, preserveOwner bool) error {
if err := os.MkdirAll(dstDir, 0755); err != nil {
return fmt.Errorf("failed to create destination dir: %w", err)
}
cmdMgr := cmd.NewCommandMgr(cmd.WithContext(ctx), cmd.WithIgnoreExist1())
cmdMgr := cmd.NewCommandMgr(cmd.WithContext(ctx))
if len(secret) != 0 {
return runTarGzDecryptToDir(cmdMgr, filePath, dstDir, secret, false)
return runTarGzDecryptToDirWithOptions(cmdMgr, filePath, dstDir, secret, false, preserveOwner)
}
return runTarGzExtractToDir(cmdMgr, filePath, dstDir)
return runTarGzExtractToDirWithOptions(cmdMgr, filePath, dstDir, preserveOwner)
}
func (t TarGzArchiver) Compress(ctx context.Context, sourcePaths []string, dstFile string, secret string) error {
@@ -79,18 +83,34 @@ func runTarGzEncryptToFile(cmdMgr *cmd.CommandHelper, dst, secret string, tarArg
}
func runTarGzExtractToDir(cmdMgr *cmd.CommandHelper, src, dst string) error {
return cmdMgr.Run("tar", "-zxvf", src, "-C", dst)
return runTarGzExtractToDirWithOptions(cmdMgr, src, dst, false)
}
func runTarGzExtractToDirWithOptions(cmdMgr *cmd.CommandHelper, src, dst string, preserveOwner bool) error {
args := []string{"-zxvf", src, "-C", dst}
if preserveOwner {
args = append([]string{"--same-owner", "--same-permissions"}, args...)
}
return cmdMgr.Run("tar", args...)
}
func runTarGzDecryptToDir(cmdMgr *cmd.CommandHelper, src, dst, secret string, withSalt bool) error {
return runTarGzDecryptToDirWithOptions(cmdMgr, src, dst, secret, withSalt, false)
}
func runTarGzDecryptToDirWithOptions(cmdMgr *cmd.CommandHelper, src, dst, secret string, withSalt, preserveOwner bool) error {
opensslArgs := []string{"enc", "-d", "-aes-256-cbc"}
if withSalt {
opensslArgs = append(opensslArgs, "-salt")
}
opensslArgs = append(opensslArgs, "-pass", "env:BACKUP_SECRET", "-in", src)
tarArgs := []string{"-zxf", "-", "-C", dst}
if preserveOwner {
tarArgs = append([]string{"--same-owner", "--same-permissions"}, tarArgs...)
}
_, err := cmdMgr.RunPipe(
cmd.PipeCommand{Name: "openssl", Args: opensslArgs, Env: []string{"BACKUP_SECRET=" + secret}},
cmd.PipeCommand{Name: "tar", Args: []string{"-zxf", "-", "-C", dst}},
cmd.PipeCommand{Name: "tar", Args: tarArgs},
)
return err
}
+1 -1
View File
@@ -45,7 +45,7 @@ func (z X7zArchiver) Compress(ctx context.Context, sourcePaths []string, dstFile
relativePaths[i] = path.Base(sp)
}
cmdArgs := append([]string{"a", "-r", tmpFile}, relativePaths...)
cmdArgs := append([]string{"a", "-snh", "-snl", tmpFile}, relativePaths...)
cmdMgr := cmd.NewCommandMgr(cmd.WithWorkDir(baseDir), cmd.WithContext(ctx))
if err = cmdMgr.Run("7z", cmdArgs...); err != nil {
return err
+10 -1
View File
@@ -20,10 +20,19 @@ func NewZipArchiver() ShellArchiver {
}
func (z ZipArchiver) Extract(ctx context.Context, filePath, dstDir string, secret string) error {
return z.ExtractWithOptions(ctx, filePath, dstDir, secret, false)
}
func (z ZipArchiver) ExtractWithOptions(ctx context.Context, filePath, dstDir, _ string, preserveOwner bool) error {
if err := checkCmdAvailability("unzip"); err != nil {
return err
}
return cmd.NewCommandMgr(cmd.WithContext(ctx)).Run("unzip", "-qo", filePath, "-d", dstDir)
args := []string{"-qo"}
if preserveOwner {
args = append(args, "-X")
}
args = append(args, filePath, "-d", dstDir)
return cmd.NewCommandMgr(cmd.WithContext(ctx)).Run("unzip", args...)
}
func (z ZipArchiver) Compress(ctx context.Context, sourcePaths []string, dstFile string, _ string) error {
+7
View File
@@ -271,3 +271,10 @@ export const getFileSharePasswordFromHash = (hash: string) => {
const hashParams = new URLSearchParams(hash.replace(/^#/, ''));
return hashParams.get(FILE_SHARE_PASSWORD_KEY)?.trim() || '';
};
export const removeFileSharePasswordFromHash = (hash: string) => {
const hashParams = new URLSearchParams(hash.replace(/^#/, ''));
hashParams.delete(FILE_SHARE_PASSWORD_KEY);
const remainingHash = hashParams.toString();
return remainingHash ? `#${remainingHash}` : '';
};
+7 -2
View File
@@ -42,7 +42,7 @@
import { checkFileShare, getPublicFileShareInfo } from '@/api/modules/files';
import { File } from '@/api/interface/file';
import i18n, { loadLocaleMessages } from '@/lang';
import { buildFileShareDownloadUrl, getFileSharePasswordFromHash } from '@/utils/file';
import { buildFileShareDownloadUrl, getFileSharePasswordFromHash, removeFileSharePasswordFromHash } from '@/utils/file';
import { dateFormat } from '@/utils/date';
import { computed, onMounted, ref } from 'vue';
import { useRoute } from 'vue-router';
@@ -86,7 +86,12 @@ const applySharedPassword = () => {
return;
}
password.value = sharedPassword;
window.history.replaceState(window.history.state, '', `${window.location.pathname}${window.location.search}`);
const remainingHash = removeFileSharePasswordFromHash(window.location.hash);
window.history.replaceState(
window.history.state,
'',
`${window.location.pathname}${window.location.search}${remainingHash}`,
);
};
const resolveBrowserLocale = () => {