fix: parallel processing to improve translation speed

translate/translate.go

@@ -2,7 +2,7 @@
  * @Author: Vincent Young
  * @Date: 2024-09-16 11:59:24
  * @LastEditors: Vincent Yang
- * @LastEditTime: 2025-01-20 17:09:59
+ * @LastEditTime: 2025-02-01 03:21:41
  * @FilePath: /DeepLX/translate/translate.go
  * @Telegram: https://t.me/missuo
  * @GitHub: https://github.com/missuo
@@ -19,6 +19,7 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
+	"sync"
 
 	"github.com/abadojack/whatlanggo"
 	"github.com/imroc/req/v3"
@@ -126,30 +127,47 @@ func TranslateByDeepLX(sourceLang, targetLang, text string, tagHandling string,
 		}, nil
 	}
 
-	// Split text by newlines and store them for later reconstruction
+	// Split text by newlines
 	textParts := strings.Split(text, "\n")
-	var translatedParts []string
-	var allAlternatives [][]string // Store alternatives for each part
 
-	for _, part := range textParts {
+	// Create channels for results
-		if strings.TrimSpace(part) == "" {
+	type translationResult struct {
-			translatedParts = append(translatedParts, "")
+		index        int
-			allAlternatives = append(allAlternatives, []string{""})
+		translation  string
-			continue
+		alternatives []string
+		err          error
+	}
+	results := make(chan translationResult, len(textParts))
+
+	// Create a wait group to track all goroutines
+	var wg sync.WaitGroup
+
+	// Launch goroutines for each text part
+	for i := range textParts {
+		wg.Add(1)
+		go func(index int, text string) {
+			defer wg.Done()
+
+			if strings.TrimSpace(text) == "" {
+				results <- translationResult{
+					index:        index,
+					translation:  "",
+					alternatives: []string{""},
+				}
+				return
 			}
 
 			// Split text first
-		splitResult, err := splitText(part, tagHandling == "html" || tagHandling == "xml", proxyURL, dlSession)
+			splitResult, err := splitText(text, tagHandling == "html" || tagHandling == "xml", proxyURL, dlSession)
 			if err != nil {
-			return DeepLXTranslationResult{
+				results <- translationResult{index: index, err: err}
-				Code:    http.StatusServiceUnavailable,
+				return
-				Message: err.Error(),
-			}, nil
 			}
 
 			// Get detected language if source language is auto
-		if sourceLang == "auto" || sourceLang == "" {
+			currentSourceLang := sourceLang
-			sourceLang = strings.ToUpper(whatlanggo.DetectLang(part).Iso6391())
+			if currentSourceLang == "auto" || currentSourceLang == "" {
+				currentSourceLang = strings.ToUpper(whatlanggo.DetectLang(text).Iso6391())
 			}
 
 			// Prepare jobs from split result
@@ -191,27 +209,7 @@ func TranslateByDeepLX(sourceLang, targetLang, text string, tagHandling string,
 
 			// Prepare translation request
 			id := getRandomNumber()
-
 			postData := &PostData{
-			Jsonrpc: "2.0",
-			Method:  "LMT_handle_jobs",
-			ID:      id,
-			Params: Params{
-				CommonJobParams: CommonJobParams{
-					Mode: "translate",
-				},
-				Lang: Lang{
-					SourceLangComputed: strings.ToUpper(sourceLang),
-					TargetLang:         strings.ToUpper(targetLangCode),
-				},
-				Jobs:      jobs,
-				Priority:  1,
-				Timestamp: getTimeStamp(getICount(part)),
-			},
-		}
-
-		if hasRegionalVariant {
-			postData = &PostData{
 				Jsonrpc: "2.0",
 				Method:  "LMT_handle_jobs",
 				ID:      id,
@@ -221,23 +219,20 @@ func TranslateByDeepLX(sourceLang, targetLang, text string, tagHandling string,
 						RegionalVariant: map[bool]string{true: targetLang, false: ""}[hasRegionalVariant],
 					},
 					Lang: Lang{
-						SourceLangComputed: strings.ToUpper(sourceLang),
+						SourceLangComputed: strings.ToUpper(currentSourceLang),
 						TargetLang:         strings.ToUpper(targetLangCode),
 					},
 					Jobs:      jobs,
 					Priority:  1,
-					Timestamp: getTimeStamp(getICount(part)),
+					Timestamp: getTimeStamp(getICount(text)),
 				},
 			}
-		}
 
 			// Make translation request
 			result, err := makeRequest(postData, "LMT_handle_jobs", proxyURL, dlSession)
 			if err != nil {
-			return DeepLXTranslationResult{
+				results <- translationResult{index: index, err: err}
-				Code:    http.StatusServiceUnavailable,
+				return
-				Message: err.Error(),
-			}, nil
 			}
 
 			// Process translation results
@@ -254,7 +249,7 @@ func TranslateByDeepLX(sourceLang, targetLang, text string, tagHandling string,
 
 				// Process alternatives
 				numBeams := len(translations[0].Get("beams").Array())
-			for i := 1; i < numBeams; i++ { // Start from 1 since 0 is the main translation
+				for i := 1; i < numBeams; i++ {
 					var altText string
 					for _, translation := range translations {
 						beams := translation.Get("beams").Array()
@@ -269,14 +264,37 @@ func TranslateByDeepLX(sourceLang, targetLang, text string, tagHandling string,
 			}
 
 			if partTranslation == "" {
-			return DeepLXTranslationResult{
+				results <- translationResult{index: index, err: fmt.Errorf("translation failed")}
-				Code:    http.StatusServiceUnavailable,
+				return
-				Message: "Translation failed",
-			}, nil
 			}
 
-		translatedParts = append(translatedParts, partTranslation)
+			results <- translationResult{
-		allAlternatives = append(allAlternatives, partAlternatives)
+				index:        index,
+				translation:  partTranslation,
+				alternatives: partAlternatives,
+			}
+		}(i, textParts[i])
+	}
+
+	// Close results channel when all goroutines are done
+	go func() {
+		wg.Wait()
+		close(results)
+	}()
+
+	// Collect results maintaining original order
+	translatedParts := make([]string, len(textParts))
+	allAlternatives := make([][]string, len(textParts))
+
+	for result := range results {
+		if result.err != nil {
+			return DeepLXTranslationResult{
+				Code:    http.StatusServiceUnavailable,
+				Message: result.err.Error(),
+			}, nil
+		}
+		translatedParts[result.index] = result.translation
+		allAlternatives[result.index] = result.alternatives
 	}
 
 	// Join all translated parts with newlines
@@ -298,9 +316,9 @@ func TranslateByDeepLX(sourceLang, targetLang, text string, tagHandling string,
 			if i < len(alts) {
 				altParts = append(altParts, alts[i])
 			} else if len(translatedParts[j]) == 0 {
-				altParts = append(altParts, "") // Keep empty lines
+				altParts = append(altParts, "")
 			} else {
-				altParts = append(altParts, translatedParts[j]) // Use main translation if no alternative
+				altParts = append(altParts, translatedParts[j])
 			}
 		}
 		combinedAlternatives = append(combinedAlternatives, strings.Join(altParts, "\n"))
@@ -308,7 +326,7 @@ func TranslateByDeepLX(sourceLang, targetLang, text string, tagHandling string,
 
 	return DeepLXTranslationResult{
 		Code:         http.StatusOK,
-		ID:           getRandomNumber(), // Using new ID for the complete translation
+		ID:           getRandomNumber(),
 		Data:         translatedText,
 		Alternatives: combinedAlternatives,
 		SourceLang:   sourceLang,