Skip to content

Instantly share code, notes, and snippets.

@Micrified
Last active January 29, 2024 22:32
Show Gist options
  • Save Micrified/569fa7db940c87951e78aadbeadbdacf to your computer and use it in GitHub Desktop.
Save Micrified/569fa7db940c87951e78aadbeadbdacf to your computer and use it in GitHub Desktop.
Parallel MD5 hasher (to be compared)
package main
// Continue here: https://go.dev/blog/pipelines
import (
"crypto/md5"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"sort"
"sync"
)
type digest struct {
Filepath string
MD5Sum [md5.Size]byte
Error error
}
func MD5All(dir string) (map[string]digest, error) {
m := make(map[string]digest)
in, done := make(chan digest), make(chan struct{})
defer close(in)
var g sync.WaitGroup
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
g.Add(1)
go func(r chan<- digest, path string, info os.FileInfo, err error) {
defer g.Done()
if nil != err {
r <- digest{Filepath: path, MD5Sum: [md5.Size]byte{}, Error: err}
return
}
if !info.Mode().IsRegular() {
return
}
data, err := ioutil.ReadFile(path)
if nil != err {
r <- digest{Filepath: path, MD5Sum: [md5.Size]byte{}, Error: err}
return
}
r <- digest{Filepath: path, MD5Sum: md5.Sum(data), Error: nil}
}(in, path, info, err)
return nil
})
// Launch routine to handle incoming
go func(done <-chan struct{}, in <-chan digest) {
for d := range in {
select {
case <- in:
m[d.Filepath] = d
case <- done:
return
}
}
}(done, in)
// Wait for all digests to be processed, then signal closure
g.Wait()
close(done)
return m, err
}
func main() {
m, err := MD5All(os.Args[1])
if nil != err {
fmt.Println(err)
return
}
var paths []string
for path := range m {
paths = append(paths, path)
}
sort.Strings(paths)
for _, path := range paths {
if m[path].Error != nil {
fmt.Printf("Error (%s) %s\n", m[path].Error, path)
} else {
fmt.Printf("%X %s\n", m[path].MD5Sum, path)
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment