[golang] 字典整合工具

博主： zmx
发布时间：2025 年 07 月 01 日
1 次浏览
暂无评论
6085字数
分类： Go

# 引言

早期使用python进行合并，但是性能太低，随后用go重构了一版，因为不太会用go，代码部分借助AI进行生成。构建基础字典数据，姓氏+生日
将多个文件合并到一起，并且进行去重排序。

# 代码

```go
package main

import (
	"bufio"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strings"
	"sync"
	"time"
	"unicode"
)

var (
	RED    = "\033[91m"
	GREEN  = "\033[92m"
	YELLOW = "\033[93m"
	RESET  = "\033[0m"

excludeExts = map[string]struct{}{
		".py": {}, ".bak": {}, ".cmd": {}, ".bat": {}, ".sh": {}, ".vbs": {}, ".ps1": {}, ".js": {},
		".html": {}, ".htm": {}, ".php": {}, ".sql": {}, ".conf": {}, ".ini": {}, ".log": {}, ".md": {},
		".xlsx": {}, ".xls": {}, ".docx": {}, ".doc": {}, ".pptx": {}, ".ppt": {}, ".pdf": {}, ".jpg": {},
		".jpeg": {}, ".png": {}, ".gif": {}, ".bmp": {}, ".ico": {}, ".exe": {},
	}
)

func generateDateBasedPasswords(days int, prefixes []string) []string {
	now := time.Now()
	passwdSet := make([]string, 0, days*len(prefixes))
	for i := 0; i < days; i++ {
		dateStr := now.AddDate(0, 0, -i).Format("20060102")
		for _, prefix := range prefixes {
			passwdSet = append(passwdSet, prefix+dateStr)
		}
	}
	return passwdSet
}
func filterAllowedChars(s string) string {
	var b strings.Builder
	for _, r := range s {
		if r >= 33 && r <= 126 {
			b.WriteRune(r)
		}
		// 如果想保留空格，可加上：
		// else if r == 32 {
		//     b.WriteRune(r)
		// }
	}
	return b.String()
}

func getFileSize(path string) (int64, error) {
	info, err := os.Stat(path)
	if err != nil {
		return 0, err
	}
	return info.Size(), nil
}

// 格式化文件大小成可读格式
func humanFileSize(size int64) string {
	const unit = 1024
	if size < unit {
		return fmt.Sprintf("%.2fB", float64(size))
	}
	div, exp := int64(unit), 0
	for n := size / unit; n >= unit; n /= unit {
		div *= unit
		exp++
	}
	units := []string{"KB", "MB", "GB", "TB"}
	return fmt.Sprintf("%.2f%s", float64(size)/float64(div), units[exp])
}

func cleanEdgesIfNeeded(s string) string {
	if len(strings.TrimSpace(s)) == 0 {
		return ""
	}
	// 判断头尾是否都是可打印字符
	if isPrintable(rune(s[0])) && isPrintable(rune(s[len(s)-1])) {
		return s
	}
	// 去除两端不可打印字符（strip + 过滤非printable）
	trimmed := strings.TrimSpace(s)
	var b strings.Builder
	for _, c := range trimmed {
		if isPrintable(c) && c != '\x0b' && c != '\x0c' {
			b.WriteRune(c)
		}
	}
	return b.String()
}

func isPrintable(r rune) bool {
	// 类似 Python 的 string.printable 判断
	if unicode.IsPrint(r) && r != '\x0b' && r != '\x0c' {
		return true
	}
	return false
}

func processFile(filePath string, minLength, maxLength int) ([]string, string, string, string) {
	start := time.Now()
	emailRegex := regexp.MustCompile(`(?i)[a-z0-9._%+\-]+@[a-z0-9.\-]+\.[a-z]{2,}`)
	file, err := os.Open(filePath)
	if err != nil {
		return nil, fmt.Sprintf("Error opening %s: %v", filepath.Base(filePath), err), "0.00", ""
	}
	defer file.Close()

var passwords []string
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if len(line) < minLength {
			continue
		}

// 处理引号
		if (strings.HasPrefix(line, "'") && strings.HasSuffix(line, "'") ||
			strings.HasPrefix(line, "\"") && strings.HasSuffix(line, "\"")) && len(line) > 1 {
			line = line[1 : len(line)-1]
		}

// 清理特殊HTML编码
		if strings.Contains(line, "&") {
			line = strings.ReplaceAll(line, "&lt;", "")
			line = strings.ReplaceAll(line, "&gt;", "")
			line = strings.ReplaceAll(line, "&amp;", "&")
			line = strings.ReplaceAll(line, "&#33;", "!")
			line = strings.ReplaceAll(line, "&#38;", "&")
			line = strings.ReplaceAll(line, "&#34;", "\"")
			line = strings.ReplaceAll(line, "&#39;", "'")
			line = strings.ReplaceAll(line, "&#60;", "<")
			line = strings.ReplaceAll(line, "&#62;", ">")
		}
		if emailRegex.MatchString(line) {
			continue // 跳过包含邮箱的行
		}
		// 清理边缘不可见字符
		line = cleanEdgesIfNeeded(line)
		line = filterAllowedChars(line)

if len(line) >= minLength && len(line) < maxLength {
			passwords = append(passwords, line)
		}
	}
	elapsed := time.Since(start).Seconds()
	size, _ := getFileSize(filePath)
	return passwords, filepath.Base(filePath), humanFileSize(size), fmt.Sprintf("%.2f", elapsed)
}
func formatWithComma(n int) string {
	s := fmt.Sprintf("%d", n)
	nStr := ""
	for i, c := range reverseString(s) {
		if i > 0 && i%3 == 0 {
			nStr = "," + nStr
		}
		nStr = string(c) + nStr
	}
	return nStr
}

func reverseString(s string) string {
	runes := []rune(s)
	for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {
		runes[i], runes[j] = runes[j], runes[i]
	}
	return string(runes)
}

func main() {
	startTime := time.Now()

aDir := `d:\tools\密码破解\Dict\`
	outputFile := filepath.Join(filepath.Dir(aDir), "new.txt")

maxWorkers := runtime.NumCPU()
	fmt.Printf("[#] Max workers: %s%d%s\n", GREEN, maxWorkers, RESET)
	fmt.Printf("[$] Start time: %s\n", time.Now().Format("2006-01-02 15:04:05"))

fmt.Println("[#] Generating date-based passwords...")
	prefixes := []string{"", "li", "lin", "duan", "zhao", "sun", "tian", "he", "an", "guo", "zhang",
		"zhou", "wu", "wang", "du", "cao", "kong", "chen", "liu", "han", "tang", "ma", "zheng"}

passwdSet := generateDateBasedPasswords(36865, prefixes)
	fmt.Printf("[#] Generated %s%s%s date+prefix passwords\n", GREEN, formatWithComma(len(passwdSet)), RESET)

// 扫描目录文件
	fmt.Println("[#] Scanning files in:", aDir)

var filePaths []string
	err := filepath.WalkDir(aDir, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}

// 如果是目录且不是根目录，跳过（不再向下递归）
		if d.IsDir() && path != aDir {
			return filepath.SkipDir
		}

if !d.IsDir() {
			ext := strings.ToLower(filepath.Ext(path))
			if _, excluded := excludeExts[ext]; !excluded {
				filePaths = append(filePaths, path)
			}
		}
		return nil
	})

if err != nil {
		fmt.Printf("[!] Error scanning directory: %v\n", err)
	}

fmt.Printf("[#] Total file number: %s%d%s\n", GREEN, len(filePaths), RESET)

// 文件按大小排序
	sort.Slice(filePaths, func(i, j int) bool {
		si, err1 := getFileSize(filePaths[i])
		sj, err2 := getFileSize(filePaths[j])
		if err1 != nil || err2 != nil {
			return false
		}
		return si < sj
	})

// 多线程处理文件
	fmt.Println("[#] Processing files with multi-threading...")

type result struct {
		passwords []string
		info      string
		size      string
		usetime   string
		index     int
	}

resultsChan := make(chan result, len(filePaths))
	var wg sync.WaitGroup
	semaphore := make(chan struct{}, maxWorkers)

for i, file := range filePaths {
		wg.Add(1)
		semaphore <- struct{}{}
		go func(idx int, f string) {
			defer wg.Done()
			passwds, info, size, usetime := processFile(f, 8, 32)
			resultsChan <- result{passwds, info, size, usetime, idx}
			<-semaphore
		}(i, file)
	}

go func() {
		wg.Wait()
		close(resultsChan)
	}()

allPasswords := make([]string, 0, 1000000)
	duplicateSize := 0

for res := range resultsChan {
		if len(res.passwords) > 0 {
			fmt.Printf("    [+] [%d / %d] [%s%ss%s / %s%.2fs%s], Extracted %s%d%s from %s %s\n",
				res.index+1, len(filePaths),
				GREEN, res.usetime, RESET,
				RED, time.Since(startTime).Seconds(), RESET,
				YELLOW, len(res.passwords), RESET,
				res.info, res.size)

allPasswords = append(allPasswords, res.passwords...)

if (res.index+1)%50 == 0 {
				before := len(allPasswords)
				allPasswords = bucketDeduplicate(allPasswords, maxWorkers)
				after := len(allPasswords)
				duplicateSize += before - after
				runtime.GC()
				fmt.Printf("\r [=] Processing, Current size %s%s%s DUP:%s%s%s\n",
					GREEN, formatWithComma(len(allPasswords)), RESET,
					RED, formatWithComma(duplicateSize), RESET)

}
		} else if strings.HasPrefix(res.info, "Error") {
			fmt.Printf("    [!] %s\n", res.info)
		}
	}
	runtime.GC()
	fmt.Printf("\n[#] Deduplicating... Current length %s%s%s, DUP:%s%s%s \n",
		GREEN, formatWithComma(len(allPasswords)), RESET,
		RED, formatWithComma(duplicateSize), RESET)
	before := len(allPasswords)
	passwdSet = bucketDeduplicate(allPasswords, maxWorkers)
	after := len(passwdSet)

percentage := float64(after) * 100 / float64(before)

fmt.Printf("[%s%.2fs%s] duplicated size: %s%s%s (%.2f%% kept)\n",
		GREEN, time.Since(startTime).Seconds(), RESET,
		RED, formatWithComma(before-after+duplicateSize), RESET,
		percentage)

fmt.Printf("[#] Total unique passwords: %s%s%s\n",
		YELLOW, formatWithComma(len(passwdSet)), RESET)

fmt.Printf("[#] Writing to file: %s... ", outputFile)
	writeStart := time.Now()
	err = os.WriteFile(outputFile, []byte(strings.Join(passwdSet, "\n")), 0644)
	if err != nil {
		fmt.Printf("Error writing output file: %v\n", err)
	} else {
		fmt.Printf("[%s%.2fs%s / %s%.2fs%s]\n",
			GREEN, time.Since(writeStart).Seconds(), RESET,
			RED, time.Since(startTime).Seconds(), RESET)
	}

fmt.Printf("[$] Finished at: %s | Duration: %s%.2fs%s\n",
		time.Now().Format("2006-01-02 15:04:05"), GREEN, time.Since(startTime).Seconds(), RESET)

}

func bucketDeduplicate(passwords []string, workers int) []string {
	buckets := make(map[byte][]string)
	otherBucket := []string{}

for i := range passwords {
		p := passwords[i]
		if len(p) == 0 {
			continue
		}
		pCopy := string([]byte(p)) // 避免与原始数据共享内存
		first := pCopy[0]

if (first >= 'a' && first <= 'z') || (first >= 'A' && first <= 'Z') || (first >= '0' && first <= '9') {
			buckets[first] = append(buckets[first], pCopy)
		} else {
			otherBucket = append(otherBucket, pCopy)
		}
		passwords[i] = "" // 释放原始引用
	}

// 构造有序桶顺序
	var keys []byte
	for ch := byte('0'); ch <= '9'; ch++ {
		if _, ok := buckets[ch]; ok {
			keys = append(keys, ch)
		}
	}
	for ch := byte('A'); ch <= 'Z'; ch++ {
		if _, ok := buckets[ch]; ok {
			keys = append(keys, ch)
		}
	}
	for ch := byte('a'); ch <= 'z'; ch++ {
		if _, ok := buckets[ch]; ok {
			keys = append(keys, ch)
		}
	}
	if len(otherBucket) > 0 {
		keys = append(keys, '#') // '#' 代表其他字符
	}

type dedupResult struct {
		bucket  byte
		passwds []string
	}

resultMap := make(map[byte][]string, len(keys))
	var mu sync.Mutex
	var wg sync.WaitGroup
	sem := make(chan struct{}, workers)

dedup := func(data []string) []string {
		set := make(map[string]struct{}, len(data))
		for _, s := range data {
			set[s] = struct{}{}
		}
		out := make([]string, 0, len(set))
		for s := range set {
			out = append(out, s)
		}
		return out
	}

for _, k := range keys {
		wg.Add(1)
		sem <- struct{}{}
		go func(bucket byte, bucketData []string) {
			defer wg.Done()
			olen := len(bucketData)
			if olen == 0 {
				return
			}
			deduped := dedup(bucketData)
			mu.Lock()
			resultMap[bucket] = deduped
			mu.Unlock()
			fmt.Printf("   [*] Deduplicating bucket '%c'... %s%d/%d%s items (%.2f%%)\n",
				bucket, GREEN, len(deduped), olen, RESET,
				float64(len(deduped))*100/float64(olen))
			<-sem
		}(k, func() []string {
			if k == '#' {
				return otherBucket
			}
			return buckets[k]
		}())
	}

wg.Wait()

finalList := make([]string, 0)
	for _, k := range keys {
		finalList = append(finalList, resultMap[k]...)
	}

return finalList
}
```

# 使用方法

先对代码进行修改`aDir` 和 `outputFile`，可以改成当前目录，自行发挥吧。然后直接执行即可。

如果觉得我的文章对你有用，请随意赞赏

发表评论取消回复
使用cookie技术保留您的个人信息以便您下次快速评论，继续评论表示您已同意该条款了昂。

评论 *

私密评论

名称 *

🎲

邮箱 *

地址

TOH ZHEN HAO .
现在被禁用了，然后我要删除可删除不了，我已通过使用cmd（管理...
TOH ZHEN HAO .
无法被删除
嘻嘻
也可以不用工具，直接以管理员身份打开reg，然后权限，高级，改...
plus
新版1.15.11是除了EXE额外加入了sys驱动里面的黑名单...
jia
谢谢，有修改好的工程文件发一份吗？我编译一下

[golang] 字典整合工具

zmx • 2025 年 07 月 01 日