Golang file duplication removal - supports multi platform Windows, Linux, MacOS

3 comments

1. Haozi
   July 21st, 2021 at 03:14 pm

   If you want to simply re shoot the pictures, you can find the blog post (only deal with the current directory), but you still do it yourself, as follows: (Both Funcs are directly copied online)

   func GetAllFile(dirpath string) ([]string, error) {
   result := []string{}
   fis, err := ioutil. ReadDir(dirpath)
   if err != nil {
   return result, err
   }
   for _, fi := range fis {
   fullname := dirpath + "/" + fi.Name()
   if fi.IsDir() {
   temp, err := GetAllFile(fullname)
   if err != nil {
   return result, err
   }
   result = append(result, temp...)
   } else {
   result = append(result, fullname)
   }
   }
   return result, nil
   }

   func Md5Sum(fullname string) (string, error) {
   f, err := os. Open(fullname)
   if err != nil {
   return "", err
   }
   defer f.Close()
   bs, err := ioutil. ReadAll(f)
   if err != nil {
   return "", err
   }
   md5 := fmt. Sprintf("%x", md5.Sum(bs))
   runtime. GC()
   return md5, nil
   }

   func main() {
   md5Map := make(map[string][]string)
   tmp := make(map[string]int)
   dirpath := "H:/JPG/"
   list, _ := GetAllFile(dirpath)
   for _, v := range list {
   md5, _ := Md5Sum(v)
   if _, ok := md5Map[md5]; ok {
   tmp[md5]++
   }
   md5Map[md5] = append(md5Map[md5], v)
   }
   for k, v := range tmp {
   fmt. Println(md5Map[k], v)
   }
   }

   reply
2. Liu Dong 19950828
   August 10th, 2021 at 11:29 pm

   Good idea. Come on, try to use a more memory saving method. My videos are too big one by one, and the calculation is very slow, but there are available tools, so I have nothing to complain about. Please continue to run, please.

   reply
   1. Chuwen
      August 12th, 2021 at 09:42 am

      Thank you for your support:)

      I can study what you said later

      reply