Fileutil

Package fileutil implements some basic functions for file operations.

Source:

https://github.com/duke-git/lancet/blob/main/fileutil/file.go

Usage:

import (
    "github.com/duke-git/lancet/v2/fileutil"
)

Index

ClearFile
CreateFile
CreateDir
CopyFile
CopyDir
CurrentPath
FileMode
MiMeType
IsExist
IsLink
IsDir
ListFileNames
RemoveFile
ReadFileToString
ReadFileByLine
Zip
ZipAppendEntry
UnZip
IsZipFile
FileSize
MTime
Sha
ReadCsvFile
WriteCsvFile
WriteMapsToCsv
WriteStringToFile
WriteBytesToFile
ReadFile
ChunkRead
ParallelChunkRead

Documentation

ClearFile

Clear the file content, write empty string to the file.

Signature:

func ClearFile(path string) error

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    err := fileutil.ClearFile("./test.txt")
    if err != nil {
        fmt.Println(err)
    }
}

CreateFile

Create file in path. return true if create succeed.

Signature:

func CreateFile(path string) bool

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    isCreatedSucceed := fileutil.CreateFile("./test.txt")
    fmt.Println(isCreatedSucceed)
}

CreateDir

Create directory in absolute path. param `absPath` like /a, /a/b.

Signature:

func CreateDir(absPath string) error

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    err := fileutil.CreateDir("/a/b") // will create folder /a/b
    fmt.Println(err)
}

CopyFile

Copy src file to dest file. If dest file exist will overwrite it.

Signature:

func CopyFile(srcPath string, dstPath string) error

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    err := fileutil.CopyFile("./test.txt", "./test_copy.txt")
    if err != nil {
        fmt.Println(err)
    }
}

CopyDir

Copy src directory to dst directory, it will copy all files and directories recursively. the access permission will be the same as the source directory. if dstPath exists, it will return an error.

Signature:

func CopyDir(srcPath string, dstPath string) error

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    err := fileutil.CopyFile("./test_src", "./test_dest")
    if err != nil {
        fmt.Println(err)
    }
}

CurrentPath

return current absolute path.

Signature:

func CurrentPath() string

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    absPath := CurrentPath()
    fmt.Println(absPath)
}

FileMode

Return file mode infomation.

Signature:

func FileMode(path string) (fs.FileMode, error)

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    mode, err := fileutil.FileMode("./test.txt")
    if err != nil {
        fmt.Println(err)
    }
    fmt.Println(mode)
}

MiMeType

Get file mime type, 'file' param's type should be string or *os.File.

Signature:

func MiMeType(file any) string

Example:Run

package main

import (
    "fmt"
    "os"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    type1 := fileutil.MiMeType("./test.txt")
    fmt.Println(type1) //text/plain; charset=utf-8

    f, _ := os.Open("./file.go")
    type2 := fileutil.MiMeType(f)
    fmt.Println(type2) //text/plain; charset=utf-8
}

IsExist

Checks if a file or directory exists.

Signature:

func IsExist(path string) bool

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    fileutil.CreateFile("./test.txt")
    isFileExist := fileutil.IsExist("./test.txt")
    fmt.Println(isFileExist) //true
}

IsLink

Checks if a file is symbol link or not.

Signature:

func IsLink(path string) bool

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    isLinkFile := fileutil.IsLink("./test.txt")
    fmt.Println(isLinkFile) //false
}

IsDir

Checks if the path is directy or not.

Signature:

func IsDir(path string) bool

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    isDir := fileutil.IsDir("./")
    fmt.Println(isDir) //true

    isDir = fileutil.IsDir("./test.txt")
    fmt.Println(isDir) //false
}

ListFileNames

List all file names in given path.

Signature:

func ListFileNames(path string) ([]string, error)

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    fileNames, _ := fileutil.ListFileNames("./")
    fmt.Println(fileNames)
}

RemoveFile

Remove the file of path.

Signature:

func RemoveFile(path string) error

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    err := fileutil.RemoveFile("./test.txt")
    if err != nil {
        fmt.Println(err)
    }
}

ReadFileToString

Return string of file content.

Signature:

func ReadFileToString(path string) (string, error)

Example:Run

package main

import (
    "fmt"
    "os"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    path := "./test.txt"
    fileutil.CreateFile(path)

    f, _ := os.OpenFile(path, os.O_WRONLY|os.O_TRUNC, 0777)
    f.WriteString("hello world")

    content, _ := fileutil.ReadFileToString(path)
    fmt.Println(content) //hello world
}

ReadFileByLine

Read file line by line, and return slice of lines

Signature:

func ReadFileByLine(path string)([]string, error)

Example:Run

package main

import (
    "fmt"
    "os"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    path := "./text.txt"
    fileutil.CreateFile(path)

    f, _ := os.OpenFile(path, os.O_WRONLY|os.O_TRUNC, 0777)
    defer f.Close()
    f.WriteString("hello\nworld")

    contents, _ := fileutil.ReadFileByLine(path)
    fmt.Println(contents) //[]string{"hello", "world"}
}

Zip

Create a zip file of fpath, fpath could be a file or a directory.

Signature:

func Zip(fpath string, destPath string) error

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    err := fileutil.Zip("./test.txt", "./test.zip")
    if err != nil {
        fmt.Println(err)
    }
}

ZipAppendEntry

Append a single file or directory by fpath to an existing zip file.

Signature:

func ZipAppendEntry(fpath string, destPath string) error

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    err := fileutil.ZipAppendEntry("./test.txt", "./test.zip")
    if err != nil {
        fmt.Println(err)
    }
}

UnZip

Unzip the file and save it to dest path.

Signature:

func UnZip(zipFile string, destPath string) error

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    err := fileutil.Zip("./test.zip", "./unzip/test.txt")
    if err != nil {
        fmt.Println(err)
    }
}

IsZipFile

Checks if file is zip file or not.

Signature:

func IsZipFile(filepath string) bool

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    isZip := fileutil.IsZipFile("./zipfile.zip")
    fmt.Println(isZip)
}

FileSize

Returns file size in bytes.

Signature:

func FileSize(path string) (int64, error)

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    size, err := fileutil.FileSize("./testdata/test.txt")

    fmt.Println(size)
    fmt.Println(err)

    // Output:
    // 20
    // <nil>
}

MTime

Returns file modified time(unix timestamp).

Signature:

func MTime(filepath string) (int64, error)

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    mtime, err := fileutil.MTime("./testdata/test.txt")

    fmt.Println(mtime)
    fmt.Println(err)

    // Output:
    // 1682391110
    // <nil>
}

Sha

returns file sha value, param `shaType` should be 1, 256 or 512.

Signature:

func Sha(filepath string, shaType ...int) (string, error)

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    sha1, err := fileutil.Sha("./testdata/test.txt", 1)
    sha256, _ := fileutil.Sha("./testdata/test.txt", 256)
    sha512, _ := fileutil.Sha("./testdata/test.txt", 512)

    fmt.Println(sha1)
    fmt.Println(sha256)
    fmt.Println(sha512)
    fmt.Println(err)

    // Output:
    // dda3cf10c5a6ff6c6659a497bf7261b287af2bc7
    // aa6d0a3fbc3442c228d606da09e0c1dc98c69a1cac3da1909199e0266171df35
    // d22aba2a1b7a2e2f512756255cc1c3708905646920cb1eb95e45b531ba74774dbbb89baebf1f716220eb9cf4908f1cfc5b2a01267704d9a59f59d77cab609870
    // <nil>
}

ReadCsvFile

Reads file content into slice.

Signature:

func ReadCsvFile(filepath string, delimiter ...rune) ([][]string, error)

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    content, err := fileutil.ReadCsvFile("./testdata/test.csv")

    fmt.Println(content)
    fmt.Println(err)

    // Output:
    // [[Bob  12  male] [Duke  14  male] [Lucy  16  female]]
    // <nil>
}

WriteCsvFile

Write content to target csv file.

Signature:

func WriteCsvFile(filepath string, records [][]string, append bool, delimiter ...rune) error

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    fpath := "./test.csv"
    fileutil.CreateFile(fpath)

    f, _ := os.OpenFile(fpath, os.O_WRONLY|os.O_TRUNC, 0777)
    defer f.Close()

    data := [][]string{
        {"Lili", "22", "female"},
        {"Jim", "21", "male"},
    }
    err := fileutil.WriteCsvFile(fpath, data, false)

    if err != nil {
        return
    }

    content, err := fileutil.ReadCsvFile(fpath)

    if err != nil {
        return
    }
    fmt.Println(content)

    // Output:
    // [[Lili 22 female] [Jim 21 male]]
}

WriteMapsToCsv

Write slice of map to csv file.

Signature:

// filepath: path of the CSV file.
// records: slice of maps to be written. the value of map should be basic type. The maps will be sorted by key in alphabeta order, then be written into csv file.
// appendToExistingFile: If true, data will be appended to the file if it exists.
// delimiter: Delimiter to use in the CSV file.
// headers: order of the csv column headers, needs to be consistent with the key of the map.
func WriteMapsToCsv(filepath string, records []map[string]any, appendToExistingFile bool, delimiter rune, headers ...[]string) error

Example:运行

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    fpath := "./test.csv"
    fileutil.CreateFile(fpath)

    f, _ := os.OpenFile(fpath, os.O_WRONLY|os.O_TRUNC, 0777)
    defer f.Close()

    records := []map[string]any{
        {"Name": "Lili", "Age": "22", "Gender": "female"},
        {"Name": "Jim", "Age": "21", "Gender": "male"},
    }

    headers := []string{"Name", "Age", "Gender"}
    err := fileutil.WriteMapsToCsv(csvFilePath, records, false, ';', headers)

    if err != nil {
        log.Fatal(err)
    }

    content, err := fileutil.ReadCsvFile(csvFilePath, ';')

    fmt.Println(content)

    // Output:
    // [[Name Age Gender] [Lili 22 female] [Jim 21 male]]
}

WriteBytesToFile

Writes bytes to target file.

Signature:

func WriteBytesToFile(filepath string, content []byte) error

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    filepath := "./bytes.txt"

    file, err := os.Create(filepath)
    if err != nil {
        return
    }

    defer file.Close()

    err = fileutil.WriteBytesToFile(filepath, []byte("hello"))
    if err != nil {
        return
    }

    content, err := fileutil.ReadFileToString(filepath)
    if err != nil {
        return
    }

    os.Remove(filepath)

    fmt.Println(content)

    // Output:
    // hello
}

WriteStringToFile

Writes string to target file.

Signature:

func WriteStringToFile(filepath string, content string, append bool) error

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    filepath := "./test.txt"

    file, err := os.Create(filepath)
    if err != nil {
        return
    }

    defer file.Close()

    err = fileutil.WriteStringToFile(filepath, "hello", true)
    if err != nil {
        return
    }

    content, err := fileutil.ReadFileToString(filepath)
    if err != nil {
        return
    }

    os.Remove(filepath)

    fmt.Println(content)

    // Output:
    // hello
}

ReadFile

Read File or URL.

Signature:

func ReadFile(path string) (reader io.ReadCloser, closeFn func(), err error)

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    reader, fn, err := fileutil.ReadFile("https://httpbin.org/robots.txt")
    if err != nil {
        return
    }
    defer fn()

    dat, err := io.ReadAll(reader)
    if err != nil {
        return
    }
    fmt.Println(string(dat))
    // Output:
    // User-agent: *
    // Disallow: /deny
}

ChunkRead

reads a block from the file at the specified offset and returns all lines within the block.

Signature :

func ChunkRead(file *os.File, offset int64, size int, bufPool *sync.Pool) ([]string, error)

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    const mb = 1024 * 1024
    const defaultChunkSizeMB = 100 

    // test1.csv file content:
    // Lili,22,female
    // Jim,21,male
    filePath := "./testdata/test1.csv"
    f, err := os.Open(filePath)
    if err != nil {
        return
    }

    defer f.Close()

    var bufPool = sync.Pool{
        New: func() interface{} {
            return make([]byte, 0, defaultChunkSizeMB*mb)
        },
    }

    lines, err := fileutil.ChunkRead(f, 0, 100, &bufPool)
    if err != nil {
        return
    }

    fmt.Println(lines[0])
    fmt.Println(lines[1])

    // Output:
    // Lili,22,female
    // Jim,21,male
}

ParallelChunkRead

Reads the file in parallel and send each chunk of lines to the specified channel.

Signature :

// filePath: file path.
// chunkSizeMB: The size of the block (in MB, the default is 100MB when set to 0). Setting it too large will be detrimental. Adjust it as appropriate.
// maxGoroutine: The number of concurrent read chunks, the number of CPU cores used when set to 0.
// linesCh: The channel used to receive the returned results.
func ParallelChunkRead(filePath string, linesCh chan<- []string, chunkSizeMB, maxGoroutine int) error

Example:Run

package main

import (
    "fmt"
    "github.com/duke-git/lancet/v2/fileutil"
)

func main() {
    const mb = 1024 * 1024
    const defaultChunkSizeMB = 100 // 默认值

    numParsers := runtime.NumCPU()

    linesCh := make(chan []string, numParsers)
    
    // test1.csv file content:
    // Lili,22,female
    // Jim,21,male
    filePath := "./testdata/test1.csv"

    go fileutil.ParallelChunkRead(filePath, linesCh, defaultChunkSizeMB, numParsers)

    var totalLines int
    for lines := range linesCh {
        totalLines += len(lines)

        for _, line := range lines {
            fmt.Println(line)
        }
    }

    fmt.Println(totalLines)

    // Output:
    // Lili,22,female
    // Jim,21,male
    // 2
}

Fileutil ​

Source: ​

Usage: ​

Index ​

Documentation ​

ClearFile ​

CreateFile ​

CreateDir ​

CopyFile ​

CopyDir ​

CurrentPath ​

FileMode ​

MiMeType ​

IsExist ​

IsLink ​

IsDir ​

ListFileNames ​

RemoveFile ​

ReadFileToString ​

ReadFileByLine ​

Zip ​

ZipAppendEntry ​

UnZip ​

IsZipFile ​

FileSize ​

MTime ​

Sha ​

ReadCsvFile ​

WriteCsvFile ​

WriteMapsToCsv ​

WriteBytesToFile ​

WriteStringToFile ​

ReadFile ​

ChunkRead ​

ParallelChunkRead ​

Fileutil

Source:

Usage:

Index

Documentation

ClearFile

CreateFile

CreateDir

CopyFile

CopyDir

CurrentPath

FileMode

MiMeType

IsExist

IsLink

IsDir

ListFileNames

RemoveFile

ReadFileToString

ReadFileByLine

Zip

ZipAppendEntry

UnZip

IsZipFile

FileSize

MTime

Sha

ReadCsvFile

WriteCsvFile

WriteMapsToCsv

WriteBytesToFile

WriteStringToFile

ReadFile

ChunkRead

ParallelChunkRead